diff --git a/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php b/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php new file mode 100644 index 00000000..0142b5c4 --- /dev/null +++ b/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php @@ -0,0 +1,320 @@ + 'control', + PriceExperiment::HIGH => 'high', + ]; + + /** + * Per-arm matured floor below which the Welch p-value (normal approximation to + * Student's t) is too anti-conservative to trust, so the verdict is withheld. + */ + private const MIN_WELCH_N = 30; + + public function __construct( + private PriceExperimentFunnelCollector $collector, + private ProportionSignificance $significance, + private WelchTTest $welch, + private SampleRatioMismatch $srm, + ) { + parent::__construct(); + } + + public function handle(): int + { + $costPerConnectionCents = (int) round(((float) $this->option('cost-per-connection')) * 100); + $report = $this->collector->collect($costPerConnectionCents); + + if ($report['startedAt'] === null) { + $this->warn('Price experiment not started — set PRICE_EXPERIMENT_STARTED_AT to begin.'); + + return self::SUCCESS; + } + + foreach ([...$this->tableLines($report), ...$this->integrityLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)] as $line) { + $this->line($line); + } + + if ($this->option('no-discord')) { + $this->info('Skipped Discord (--no-discord).'); + + return self::SUCCESS; + } + + $webhookUrl = config('services.discord.ai_cohort_webhook_url') + ?: config('services.discord.webhook_url'); + + (new DiscordWebhook($webhookUrl))->send('', [$this->buildEmbed($report)]); + + $this->info('Price experiment funnel report sent to Discord.'); + + return self::SUCCESS; + } + + /** + * @param array{currency: string, revenueAvailable: bool, variants: array>} $report + * @return list + */ + private function tableLines(array $report): array + { + $currency = $report['currency']; + $lines = [sprintf( + '%-8s %5s %5s %5s %5s %5s %6s %8s %8s %8s %8s %6s', + 'Variant', 'Assg', 'Actd', 'Card', 'MatU', 'Conv', 'Conv%', 'MRR', 'Cost', 'CM', 'CM/u', 'C/Pyr', + )]; + + foreach (self::LABELS as $key => $label) { + $row = $report['variants'][$key]; + $mature = $row['assignedMature'] > 0; + $showMoney = $report['revenueAvailable'] && $mature; + + $lines[] = sprintf( + '%-8s %5d %5d %5d %5d %5d %6s %8s %8s %8s %8s %6s', + $label, + $row['assigned'], + $row['activated'], + $row['subscribed'], + $row['assignedMature'], + $row['convertedMature'], + $mature ? ((int) round($row['conversionRate'] * 100)).'%' : 'pend', + $showMoney ? Money::format($row['mrrCents'], $currency) : '—', + $mature ? Money::format($row['costCents'], $currency) : '—', + $showMoney ? Money::format($row['contributionMarginCents'], $currency) : '—', + $showMoney && $row['cmPerAssignedCents'] !== null ? Money::format($row['cmPerAssignedCents'], $currency) : '—', + $row['connectionsPerPayer'] !== null ? number_format($row['connectionsPerPayer'], 1) : '—', + ); + } + + return $lines; + } + + /** + * Revenue-map integrity warnings, surfaced in the report the decision-maker + * reads (not just the logs): Stripe unreachable, or net-active subscriptions on + * price ids missing from the monthly-equivalent map — which would silently + * undercount MRR/CM as 0 and bias the comparison against whichever arm's price + * is unmapped. Empty (no lines) when everything resolves. + * + * @param array{revenueAvailable: bool, unmappedPriceIds: list} $report + * @return list + */ + private function integrityLines(array $report): array + { + $lines = []; + + if (! $report['revenueAvailable']) { + $lines[] = ''; + $lines[] = '⚠ REVENUE UNAVAILABLE — Stripe prices could not be loaded; MRR/CM are blank this run.'; + } + + if ($report['unmappedPriceIds'] !== []) { + $lines[] = ''; + $lines[] = '⚠ UNMAPPED PRICES — net-active subs on price ids absent from the revenue map ' + .'(MRR undercounted as 0): '.implode(', ', $report['unmappedPriceIds']) + .'. Confirm both arm prices are synced under the pro product (stripe:sync-prices).'; + } + + return $lines; + } + + /** + * Sample-ratio-mismatch check: the split must be ~50/50. A low p means the + * assignment (or the pipeline that filters it) is broken, which invalidates + * every comparison below — so this is checked first, on the raw assigned counts + * and on the matured subset (to catch differential attrition). + * + * @param array{variants: array>} $report + * @return list + */ + private function srmLines(array $report): array + { + $control = $report['variants'][PriceExperiment::CONTROL]; + $high = $report['variants'][PriceExperiment::HIGH]; + $lines = ['', 'SRM (assignment balance, expect 50/50 — a low p means the split is broken):']; + + foreach (['assigned' => 'assigned', 'assignedMature' => 'matured'] as $field => $label) { + $c = (int) $control[$field]; + $h = (int) $high[$field]; + + if ($c + $h === 0) { + $lines[] = sprintf(' %-8s control %d / high %d (n=0)', $label, $c, $h); + + continue; + } + + $srm = $this->srm->evenSplit($c, $h); + $flag = $srm['p'] < 0.01 ? ' ⚠ IMBALANCE — investigate before trusting results' : ''; + $lines[] = sprintf(' %-8s control %d / high %d χ²=%.2f p=%.3f%s', $label, $c, $h, $srm['chiSq'], $srm['p'], $flag); + } + + return $lines; + } + + /** + * Primary decision test — Welch on contribution-margin-per-user (control vs + * high), the metric we maximise — plus the conversion guardrail (Wilson CIs + + * Fisher exact, a single control-vs-treatment contrast, so no multiplicity + * correction). CM/user is the number to act on; conversion protects the funnel. + * + * @param array{revenueAvailable: bool, currency: string, variants: array>} $report + * @return list + */ + private function significanceLines(array $report): array + { + $currency = $report['currency']; + $control = $report['variants'][PriceExperiment::CONTROL]; + $high = $report['variants'][PriceExperiment::HIGH]; + + $lines = ['', 'PRIMARY — contribution margin per assigned user (Welch, high vs control):']; + + if (! $report['revenueAvailable'] || $control['assignedMature'] < 2 || $high['assignedMature'] < 2) { + $lines[] = ' pend — need mature volume in both arms and Stripe revenue to compare.'; + } else { + foreach (['control' => $control, 'high' => $high] as $label => $row) { + $lines[] = sprintf(' %-8s %8s/user (n=%d)', $label, Money::format((int) $row['cmPerAssignedCents'], $currency), $row['assignedMature']); + } + + $welch = $this->welch->compare( + $this->cmMean($high), $this->cmVariance($high), $high['assignedMature'], + $this->cmMean($control), $this->cmVariance($control), $control['assignedMature'], + ); + $delta = Money::format((int) round($welch['diff']), $currency); + $minN = min((int) $control['assignedMature'], (int) $high['assignedMature']); + + if ($minN < self::MIN_WELCH_N) { + // The p-value uses a normal approximation to Student's t; below a + // per-arm floor its tails are too thin (anti-conservative), so we + // show the effect size but withhold the verdict rather than risk a + // small-sample false positive. + $lines[] = sprintf( + ' Δ high−control: %s/user t=%.2f (small sample: min n=%d < %d → p unreliable, verdict withheld)', + $delta, $welch['t'], $minN, self::MIN_WELCH_N, + ); + } else { + $significant = $welch['p'] < 0.05; + $lines[] = sprintf( + ' Δ high−control: %s/user t=%.2f p=%.3f %s α=0.05 -> %s.%s', + $delta, $welch['t'], $welch['p'], $significant ? '<' : '≥', + $significant ? 'significant' : 'not significant', + $significant ? '' : ' Keep running.', + ); + } + } + + $lines[] = ''; + $lines[] = 'GUARDRAIL — conversion (95% Wilson CI on Conv%, n = MatU):'; + $arms = []; + + foreach (self::LABELS as $key => $label) { + $row = $report['variants'][$key]; + $n = (int) $row['assignedMature']; + $k = (int) $row['convertedMature']; + + if ($n <= 0) { + $lines[] = sprintf(' %-8s pend (n=0)', $label); + + continue; + } + + [$lower, $upper] = $this->significance->wilsonInterval($k, $n); + $lines[] = sprintf(' %-8s %6s [%6s – %6s] (n=%d)', $label, $this->percent($k / $n), $this->percent($lower), $this->percent($upper), $n); + $arms[$key] = new BinomialProportion($label, $k, $n); + } + + if (count($arms) === 2) { + $result = $this->significance->compare($arms[PriceExperiment::HIGH], $arms[PriceExperiment::CONTROL], comparisons: 1); + $lines[] = sprintf( + ' Fisher exact p=%.3f %s α=%.3f -> %s.', + $result['fisherP'], $result['significant'] ? '<' : '≥', $result['alpha'], + $result['significant'] ? 'significant' : 'not significant', + ); + } + + return $lines; + } + + /** + * @param array $row + */ + private function cmMean(array $row): float + { + return $row['assignedMature'] > 0 ? (float) $row['contributionMarginCents'] / $row['assignedMature'] : 0.0; + } + + /** + * Sample variance (n−1) of per-user CM, from the accumulated moments. + * + * @param array $row + */ + private function cmVariance(array $row): float + { + $n = (int) $row['assignedMature']; + + if ($n < 2) { + return 0.0; + } + + $sum = (float) $row['contributionMarginCents']; + + return max(0.0, ((float) $row['cmSumSq'] - ($sum * $sum) / $n) / ($n - 1)); + } + + private function percent(float $rate): string + { + return number_format($rate * 100, 1).'%'; + } + + /** + * @param array{startedAt: CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, decisionWindowDays: int, unmappedPriceIds: list, variants: array>} $report + * @return array + */ + private function buildEmbed(array $report): array + { + return [ + 'title' => '💶 Price Experiment — Funnel (control vs high)', + 'description' => "```\n".implode("\n", [...$this->tableLines($report), ...$this->integrityLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)])."\n```", + 'color' => 0x57F287, + 'fields' => [ + [ + 'name' => 'Started', + 'value' => $report['startedAt']->format('D, d M Y').sprintf(' · new signups split 50/50; both arms mature at %d days.', $report['decisionWindowDays']), + 'inline' => false, + ], + [ + 'name' => 'Legend', + 'value' => sprintf( + 'Assg = signups · Actd = activated (bank or AI = cost triggered) · Card = started a subscription · MatU = matured assigned (old enough to score) · Conv%% = ever-charged net of refund ÷ MatU (time-invariant) · MRR = monthly run-rate of currently-paying subs · Cost = active connections × %s · CM = MRR − Cost · CM/u = CM ÷ MatU (the decision metric) · C/Pyr = active connections per payer.', + Money::format($report['costPerConnectionCents'], $report['currency']), + ), + 'inline' => false, + ], + [ + 'name' => '⚠️ MONITORING ONLY', + 'value' => "**Do NOT call a winner from this weekly report.** Decide at the pre-registered horizon N (from the power analysis), not the first week Fisher/Welch crosses α — weekly peeking inflates the false-positive rate. The primary metric is **CM per assigned user** (Welch); conversion is a guardrail. CM/user is sub-cent per week at current volume, so early weeks read as 'not significant' — that is expected. Don't add/remove arms or change the split mid-flight. A 1-month CM cannot see price-driven churn, so re-read retention/LTV before permanently locking the price.", + 'inline' => false, + ], + ], + ]; + } +} diff --git a/app/Console/Commands/SyncStripePricesCommand.php b/app/Console/Commands/SyncStripePricesCommand.php index ce34f821..0d63df35 100644 --- a/app/Console/Commands/SyncStripePricesCommand.php +++ b/app/Console/Commands/SyncStripePricesCommand.php @@ -17,7 +17,7 @@ class SyncStripePricesCommand extends Command public function handle(): int { - $plans = config('subscriptions.plans', []); + $plans = $this->plansToSync(); $currency = config('cashier.currency', 'eur'); $dryRun = $this->option('dry-run'); @@ -103,6 +103,33 @@ class SyncStripePricesCommand extends Command return Command::SUCCESS; } + /** + * The base plans plus the price-experiment variant tiers, flattened into one + * list of pseudo-plans so each gets its own Stripe price + lookup key. The + * variant billing period and name are inherited from the matching base plan. + * + * @return array> + */ + private function plansToSync(): array + { + $plans = (array) config('subscriptions.plans', []); + + foreach ((array) config('subscriptions.price_experiment.variants', []) as $variantKey => $variantPlans) { + foreach ((array) $variantPlans as $planKey => $spec) { + $base = $plans[$planKey] ?? []; + + $plans["price_experiment.{$variantKey}.{$planKey}"] = [ + 'name' => ($base['name'] ?? ucfirst((string) $planKey))." (price: {$variantKey})", + 'price' => $spec['price'], + 'billing_period' => $base['billing_period'] ?? ($planKey === 'yearly' ? 'year' : 'month'), + 'stripe_lookup_key' => $spec['lookup'] ?? null, + ]; + } + } + + return $plans; + } + /** * @return Price|null */ diff --git a/app/Features/PriceExperiment.php b/app/Features/PriceExperiment.php new file mode 100644 index 00000000..94dbe935 --- /dev/null +++ b/app/Features/PriceExperiment.php @@ -0,0 +1,66 @@ +created_at?->lt(CarbonImmutable::parse($startedAt))) { + return self::LEGACY; + } + + return self::bucket((string) $user->getKey()); + } + + /** + * Deterministic, evenly-split bucket for a post-start user. The 'price:' salt + * decouples this split from every other crc32-based experiment on the same id + * (e.g. SubscriptionExperiment), so the two arms are independent. Keep this the + * single source of truth — the funnel report mirrors it to attribute users + * without reading Pennant per row. + */ + public static function bucket(string $key): string + { + return crc32('price:'.$key) % 2 === 0 ? self::CONTROL : self::HIGH; + } +} diff --git a/app/Http/Controllers/SubscriptionController.php b/app/Http/Controllers/SubscriptionController.php index 41465701..c7cc51b0 100644 --- a/app/Http/Controllers/SubscriptionController.php +++ b/app/Http/Controllers/SubscriptionController.php @@ -72,7 +72,10 @@ class SubscriptionController extends Controller abort(400, 'Invalid plan selected'); } - $priceId = $this->resolvePriceIdByLookupKey($plan['stripe_lookup_key']); + // The price tier is resolved server-side from the user's assigned variant, + // never from the request, so a user can't pick the cheaper price. + $lookupKey = $this->experimentOffer->lookupKeyFor($request->user(), $planKey); + $priceId = $this->resolvePriceIdByLookupKey($lookupKey); $subscriptionBuilder = $request->user() ->newSubscription('default', $priceId); diff --git a/app/Http/Middleware/HandleInertiaRequests.php b/app/Http/Middleware/HandleInertiaRequests.php index d39085cd..737c7989 100644 --- a/app/Http/Middleware/HandleInertiaRequests.php +++ b/app/Http/Middleware/HandleInertiaRequests.php @@ -6,9 +6,11 @@ use App\Enums\BankingConnectionStatus; use App\Enums\BankingProvider; use App\Features\CalculateBalancesOnImport; use App\Features\Mcp; +use App\Features\PriceExperiment; use App\Jobs\PurgeResidualEncryptionArtifactsJob; use App\Models\BankingConnection; use App\Services\CurrencyOptions; +use App\Services\Subscriptions\ExperimentOffer; use Illuminate\Foundation\Inspiring; use Illuminate\Http\Request; use Inertia\Middleware; @@ -16,7 +18,10 @@ use Laravel\Pennant\Feature; class HandleInertiaRequests extends Middleware { - public function __construct(private CurrencyOptions $currencyOptions) {} + public function __construct( + private CurrencyOptions $currencyOptions, + private ExperimentOffer $experimentOffer, + ) {} /** * The root template that's loaded on the first page visit. @@ -66,6 +71,18 @@ class HandleInertiaRequests extends Middleware PurgeResidualEncryptionArtifactsJob::dispatch($user); } + // Warm every Pennant feature this response reads in a single round-trip, so + // the per-feature reads below (the price-experiment pricing variant and the + // feature flags) hit Pennant's in-memory cache instead of each issuing its + // own select + insert. + if ($user) { + Feature::for($user)->values([ + CalculateBalancesOnImport::class, + Mcp::class, + PriceExperiment::class, + ]); + } + return [ ...parent::share($request), 'flash' => [ @@ -95,7 +112,9 @@ class HandleInertiaRequests extends Middleware 'subscriptionsEnabled' => config('subscriptions.enabled', false), 'aiCategorizationUpsellRate' => (int) config('ai_categorization.upsell_sample_rate'), 'pricing' => [ - 'plans' => config('subscriptions.plans', []), + 'plans' => $user + ? $this->experimentOffer->pricingPlansFor($user) + : config('subscriptions.plans', []), 'defaultPlan' => config('subscriptions.default_plan', 'monthly'), 'bestValuePlan' => config('subscriptions.best_value_plan', null), 'promo' => config('subscriptions.promo', []), diff --git a/app/Services/Stats/ExperimentFunnelCollector.php b/app/Services/Stats/ExperimentFunnelCollector.php index 9a0ca74d..090ef9e5 100644 --- a/app/Services/Stats/ExperimentFunnelCollector.php +++ b/app/Services/Stats/ExperimentFunnelCollector.php @@ -5,9 +5,7 @@ namespace App\Services\Stats; use App\Features\SubscriptionExperiment; use App\Models\User; use Carbon\CarbonImmutable; -use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Log; -use Laravel\Cashier\Cashier; use Laravel\Cashier\Subscription; class ExperimentFunnelCollector @@ -18,6 +16,8 @@ class ExperimentFunnelCollector */ private const SETTLE_BUFFER_DAYS = 3; + public function __construct(private MonthlyEquivalentPrices $prices) {} + /** * Per-variant funnel for the trial/pricing experiment. Users are attributed * by SubscriptionExperiment::bucket() — the deterministic crc32 split that is @@ -94,7 +94,7 @@ class ExperimentFunnelCollector $now = CarbonImmutable::now('UTC'); $excluded = (array) config('ai_suggestions.report.excluded_emails', []); $windows = $this->decisionWindows(); - $monthlyEquiv = $this->monthlyEquivByPriceId(); + $monthlyEquiv = $this->prices->map(); $missingPrices = []; User::query() @@ -270,70 +270,6 @@ class ExperimentFunnelCollector ]; } - /** - * Monthly-equivalent amount (in cents) for each plan price id, from Stripe. - * Yearly prices are divided by 12. Fetched by product so that archived, - * rotated price ids (Stripe mints a new id and transfers the lookup key on - * any amount change) still resolve — otherwise subscriptions on an old id - * would silently contribute 0 to MRR. Falls back to the current lookup keys - * when no product is configured. Foreign-currency and one-off prices are - * skipped. Cached for an hour; returns [] (revenue unavailable) if Stripe - * can't be reached, without caching the failure. - * - * @return array - */ - private function monthlyEquivByPriceId(): array - { - $key = 'experiment_funnel_monthly_equiv'; - - if (Cache::has($key)) { - return Cache::get($key); - } - - $productId = config('subscriptions.products.pro'); - $lookups = array_values(array_filter([ - config('subscriptions.plans.monthly.stripe_lookup_key'), - config('subscriptions.plans.yearly.stripe_lookup_key'), - ])); - - if ($productId === null && $lookups === []) { - return []; - } - - $params = $productId !== null - ? ['product' => $productId, 'limit' => 100] - : ['lookup_keys' => $lookups, 'limit' => 10]; - - try { - $prices = Cashier::stripe()->prices->all($params); - } catch (\Throwable) { - return []; - } - - $currency = strtolower((string) config('cashier.currency', 'eur')); - $map = []; - foreach ($prices->data as $price) { - if ($price->recurring === null) { - continue; - } - - if (strtolower((string) ($price->currency ?? $currency)) !== $currency) { - continue; - } - - $amount = (int) ($price->unit_amount ?? 0); - $map[$price->id] = ($price->recurring->interval ?? 'month') === 'year' - ? (int) round($amount / 12) - : $amount; - } - - if ($map !== []) { - Cache::put($key, $map, now()->addHour()); - } - - return $map; - } - /** * Days from signup until each variant's outcome can be scored: the trial * length (the longer of the two reduced trials) or the refund window. diff --git a/app/Services/Stats/MonthlyEquivalentPrices.php b/app/Services/Stats/MonthlyEquivalentPrices.php new file mode 100644 index 00000000..ab3132f0 --- /dev/null +++ b/app/Services/Stats/MonthlyEquivalentPrices.php @@ -0,0 +1,77 @@ + + */ + public function map(): array + { + if (Cache::has(self::CACHE_KEY)) { + return Cache::get(self::CACHE_KEY); + } + + $productId = config('subscriptions.products.pro'); + $lookups = array_values(array_filter([ + config('subscriptions.plans.monthly.stripe_lookup_key'), + config('subscriptions.plans.yearly.stripe_lookup_key'), + ])); + + if ($productId === null && $lookups === []) { + return []; + } + + $params = $productId !== null + ? ['product' => $productId, 'limit' => 100] + : ['lookup_keys' => $lookups, 'limit' => 10]; + + try { + $prices = Cashier::stripe()->prices->all($params); + } catch (\Throwable) { + return []; + } + + $currency = strtolower((string) config('cashier.currency', 'eur')); + $map = []; + foreach ($prices->data as $price) { + if ($price->recurring === null) { + continue; + } + + if (strtolower((string) ($price->currency ?? $currency)) !== $currency) { + continue; + } + + $amount = (int) ($price->unit_amount ?? 0); + $map[$price->id] = ($price->recurring->interval ?? 'month') === 'year' + ? (int) round($amount / 12) + : $amount; + } + + if ($map !== []) { + Cache::put(self::CACHE_KEY, $map, now()->addHour()); + } + + return $map; + } +} diff --git a/app/Services/Stats/Normal.php b/app/Services/Stats/Normal.php new file mode 100644 index 00000000..72f7bd22 --- /dev/null +++ b/app/Services/Stats/Normal.php @@ -0,0 +1,28 @@ +, + * variants: array + * } + */ + public function collect(int $costPerConnectionCents = 40): array + { + $startedValue = config('subscriptions.price_experiment.started_at'); + $startedAt = $startedValue !== null ? CarbonImmutable::parse($startedValue) : null; + $currency = strtoupper((string) config('cashier.currency', 'eur')); + $window = (int) config('subscriptions.plans.monthly.trial_days', 15) + self::SETTLE_BUFFER_DAYS; + + $variants = [ + PriceExperiment::CONTROL => $this->emptyRow(), + PriceExperiment::HIGH => $this->emptyRow(), + ]; + + if ($startedAt === null) { + return [ + 'startedAt' => null, + 'currency' => $currency, + 'revenueAvailable' => false, + 'costPerConnectionCents' => $costPerConnectionCents, + 'decisionWindowDays' => $window, + 'unmappedPriceIds' => [], + 'variants' => $variants, + ]; + } + + $now = CarbonImmutable::now('UTC'); + $excluded = (array) config('ai_suggestions.report.excluded_emails', []); + $monthlyEquiv = $this->prices->map(); + $missingPrices = []; + + User::query() + ->withTrashed() + ->where('users.created_at', '>=', $startedAt) + ->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded)) + ->with(['subscriptions' => fn ($query) => $query->where('type', 'default')]) + ->select(['id', 'created_at']) + ->withCount([ + // "Ever connected" drives activation (cost was triggered at some point). + 'bankingConnections as connection_count' => fn ($query) => $query->withTrashed(), + // "Currently active" drives the monthly recurring cost. + 'bankingConnections as active_connection_count', + 'aiConsents as ai_consent_count', + ]) + ->chunkById(500, function ($users) use (&$variants, &$missingPrices, $window, $now, $monthlyEquiv, $costPerConnectionCents): void { + foreach ($users as $user) { + $variant = PriceExperiment::bucket((string) $user->id); + + if (! isset($variants[$variant])) { + continue; + } + + $row = &$variants[$variant]; + + $row['assigned']++; + + $everConnections = (int) ($user->connection_count ?? 0); + $activeConnections = (int) ($user->active_connection_count ?? 0); + $activated = $everConnections > 0 || (int) ($user->ai_consent_count ?? 0) > 0; + + if ($activated) { + $row['activated']++; + } + + /** @var Subscription|null $subscription */ + $subscription = $user->subscriptions->sortByDesc('created_at')->first(); + $status = $subscription?->stripe_status; + $netActive = $status === 'active' && $subscription->refunded_at === null; + + // Time-invariant "ever charged, net of refund": comparable across + // cohorts of different ages. Excludes trial-only cancels. + $converted = $subscription !== null + && $subscription->refunded_at === null + && $status !== 'trialing' + && ( + $subscription->trial_ends_at === null + || $subscription->ends_at === null + || $subscription->ends_at->greaterThan($subscription->trial_ends_at) + ); + + if ($subscription !== null) { + $row['subscribed']++; + } + + $mature = CarbonImmutable::parse($user->created_at) + ->addDays($window) + ->lessThanOrEqualTo($now); + + if (! $mature) { + unset($row); + + continue; + } + + $row['assignedMature']++; + + if ($activated) { + $row['activatedMature']++; + } + + if ($converted) { + $row['convertedMature']++; + } + + $costCents = $activeConnections * $costPerConnectionCents; + $row['costCents'] += $costCents; + + $revenueCents = 0; + if ($netActive) { + $row['activeMature']++; + $priceId = (string) $subscription->stripe_price; + + if ($monthlyEquiv !== [] && ! isset($monthlyEquiv[$priceId])) { + $missingPrices[$priceId] = true; + } + + $revenueCents = (int) ($monthlyEquiv[$priceId] ?? 0); + $row['mrrCents'] += $revenueCents; + $row['payerCount']++; + $row['payerConnectionSum'] += $activeConnections; + } + + // Per-user monthly CM; cmSumSq feeds the Welch variance. The sum + // equals mrrCents − costCents, so it is not stored separately. + $cmUser = $revenueCents - $costCents; + $row['cmSumSq'] += $cmUser * $cmUser; + + unset($row); + } + }); + + if ($missingPrices !== []) { + Log::warning('Price experiment funnel: net-active subscriptions on prices absent from the monthly-equivalent map — their MRR is undercounted as 0.', [ + 'price_ids' => array_keys($missingPrices), + ]); + } + + foreach ($variants as $key => $row) { + $n = $row['assignedMature']; + $variants[$key]['conversionRate'] = $n > 0 ? (float) $row['convertedMature'] / $n : null; + $variants[$key]['contributionMarginCents'] = $row['mrrCents'] - $row['costCents']; + $variants[$key]['cmPerAssignedCents'] = $n > 0 + ? (int) round(($row['mrrCents'] - $row['costCents']) / $n) + : null; + $variants[$key]['connectionsPerPayer'] = $row['payerCount'] > 0 + ? (float) $row['payerConnectionSum'] / $row['payerCount'] + : null; + } + + return [ + 'startedAt' => $startedAt, + 'currency' => $currency, + 'revenueAvailable' => $monthlyEquiv !== [], + 'costPerConnectionCents' => $costPerConnectionCents, + 'decisionWindowDays' => $window, + 'unmappedPriceIds' => array_keys($missingPrices), + 'variants' => $variants, + ]; + } + + /** + * @return array{assigned: int, activated: int, subscribed: int, assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int, payerCount: int, payerConnectionSum: int, conversionRate: ?float, mrrCents: int, costCents: int, contributionMarginCents: int, cmPerAssignedCents: ?int, cmSumSq: float, connectionsPerPayer: ?float} + */ + private function emptyRow(): array + { + return [ + 'assigned' => 0, + 'activated' => 0, + 'subscribed' => 0, + 'assignedMature' => 0, + 'activatedMature' => 0, + 'convertedMature' => 0, + 'activeMature' => 0, + 'payerCount' => 0, + 'payerConnectionSum' => 0, + 'conversionRate' => null, + 'mrrCents' => 0, + 'costCents' => 0, + 'contributionMarginCents' => 0, + 'cmPerAssignedCents' => null, + 'cmSumSq' => 0.0, + 'connectionsPerPayer' => null, + ]; + } +} diff --git a/app/Services/Stats/SampleRatioMismatch.php b/app/Services/Stats/SampleRatioMismatch.php new file mode 100644 index 00000000..729c0c3a --- /dev/null +++ b/app/Services/Stats/SampleRatioMismatch.php @@ -0,0 +1,35 @@ + 0.0, 'p' => 1.0]; + } + + $expected = $total / 2.0; + $chiSq = (($countA - $expected) ** 2 + ($countB - $expected) ** 2) / $expected; + + // χ² with 1 df is Z²; its upper tail is P(|Z| > √χ²). + $p = 2.0 * (1.0 - Normal::cdf(sqrt($chiSq))); + + return ['chiSq' => $chiSq, 'p' => $p]; + } +} diff --git a/app/Services/Stats/WelchTTest.php b/app/Services/Stats/WelchTTest.php new file mode 100644 index 00000000..d5eec454 --- /dev/null +++ b/app/Services/Stats/WelchTTest.php @@ -0,0 +1,45 @@ + 0 ? $varA / $nA : 0.0) + ($nB > 0 ? $varB / $nB : 0.0); + $se = sqrt($seSq); + $diff = $meanA - $meanB; + + if ($se <= 0.0 || $nA < 2 || $nB < 2) { + return ['diff' => $diff, 'se' => $se, 't' => 0.0, 'df' => 0.0, 'p' => 1.0]; + } + + $t = $diff / $se; + $df = ($seSq ** 2) / ( + (($varA / $nA) ** 2) / ($nA - 1) + + (($varB / $nB) ** 2) / ($nB - 1) + ); + $p = 2.0 * (1.0 - Normal::cdf(abs($t))); + + return ['diff' => $diff, 'se' => $se, 't' => $t, 'df' => $df, 'p' => $p]; + } +} diff --git a/app/Services/Subscriptions/ExperimentOffer.php b/app/Services/Subscriptions/ExperimentOffer.php index 1110c8d3..383305c1 100644 --- a/app/Services/Subscriptions/ExperimentOffer.php +++ b/app/Services/Subscriptions/ExperimentOffer.php @@ -2,6 +2,7 @@ namespace App\Services\Subscriptions; +use App\Features\PriceExperiment; use App\Features\SubscriptionExperiment; use App\Models\User; use Carbon\CarbonImmutable; @@ -9,10 +10,11 @@ use Laravel\Cashier\Subscription; use Laravel\Pennant\Feature; /** - * Translates a user's experiment variant into the concrete offer they get: - * how many trial days per plan, whether they pay immediately, and whether they - * can still trigger the self-service refund. Single source of truth shared by - * the checkout, the paywall and the billing settings screen. + * Translates a user's experiment variants into the concrete offer they get: the + * plan prices (price experiment), how many trial days per plan, whether they pay + * immediately, and whether they can still trigger the self-service refund. Single + * source of truth shared by the checkout, the paywall and the billing settings + * screen, so a user is always shown exactly the price and terms they are charged. */ class ExperimentOffer { @@ -21,6 +23,49 @@ class ExperimentOffer return Feature::for($user)->value(SubscriptionExperiment::class); } + public function priceVariantFor(User $user): string + { + return Feature::for($user)->value(PriceExperiment::class); + } + + /** + * The plans config with the user's price-experiment variant applied: price, + * original_price and Stripe lookup key swapped for the assigned tier. Control + * and legacy users get the unchanged config. Drives both the paywall display + * and the checkout charge, so the shown price always equals the charged one. + * + * @return array> + */ + public function pricingPlansFor(User $user): array + { + $plans = (array) config('subscriptions.plans', []); + $overrides = (array) config('subscriptions.price_experiment.variants.'.$this->priceVariantFor($user), []); + + foreach ($overrides as $planKey => $override) { + if (! isset($plans[$planKey])) { + continue; + } + + $plans[$planKey]['price'] = $override['price']; + $plans[$planKey]['original_price'] = $override['original_price'] ?? null; + $plans[$planKey]['stripe_lookup_key'] = $override['lookup']; + } + + return $plans; + } + + /** + * Stripe lookup key to charge for the given plan and the user's price variant. + * Falls back to the control plan's key for control/legacy users. + */ + public function lookupKeyFor(User $user, string $planKey): string + { + return (string) config( + "subscriptions.price_experiment.variants.{$this->priceVariantFor($user)}.{$planKey}.lookup", + config("subscriptions.plans.{$planKey}.stripe_lookup_key"), + ); + } + /** * Trial days to apply at checkout for the given plan and the user's variant. */ diff --git a/config/subscriptions.php b/config/subscriptions.php index bdaf4005..ffda6575 100644 --- a/config/subscriptions.php +++ b/config/subscriptions.php @@ -45,6 +45,43 @@ return [ 'pay_now_refund_window_days' => (int) env('SUBSCRIPTION_EXPERIMENT_REFUND_WINDOW_DAYS', 3), ], + /* + |-------------------------------------------------------------------------- + | Price Experiment + |-------------------------------------------------------------------------- + | + | A/B test on the *price* of the paid plan, independent of the trial + | experiment above. Users who register on or after `started_at` are split + | evenly into `control` (the plans.* prices) and `high` (the variant prices + | below); earlier users stay "legacy" and keep the control price. While + | `started_at` is null the experiment is off and everyone gets control, so + | this is a no-op until activated. Each variant needs its own Stripe prices — + | run `php artisan stripe:sync-prices` before setting `started_at`. The yearly + | price is the monthly × 6, matching the control plan structure. + | + */ + + 'price_experiment' => [ + 'started_at' => env('PRICE_EXPERIMENT_STARTED_AT'), + // Once a winner is chosen, set this to control / high to give every user + // that price and end the split (env-only, no deploy). + 'force_variant' => env('PRICE_EXPERIMENT_FORCE_VARIANT'), + 'variants' => [ + 'high' => [ + 'monthly' => [ + 'price' => (float) env('PRICE_EXPERIMENT_HIGH_MONTHLY', 8.99), + 'original_price' => null, + 'lookup' => env('PRICE_EXPERIMENT_HIGH_MONTHLY_LOOKUP_KEY', 'whisper_pro_monthly_v9'), + ], + 'yearly' => [ + 'price' => (float) env('PRICE_EXPERIMENT_HIGH_YEARLY', 53.94), + 'original_price' => (float) env('PRICE_EXPERIMENT_HIGH_YEARLY_ORIGINAL', 107.88), + 'lookup' => env('PRICE_EXPERIMENT_HIGH_YEARLY_LOOKUP_KEY', 'whisper_pro_yearly_v9'), + ], + ], + ], + ], + /* |-------------------------------------------------------------------------- | Stripe Product IDs diff --git a/routes/console.php b/routes/console.php index 2b195663..e6e657a3 100644 --- a/routes/console.php +++ b/routes/console.php @@ -17,3 +17,4 @@ Schedule::command('stats:daily-report')->dailyAt('09:00')->timezone('Europe/Madr Schedule::command('stats:ai-cohort-report')->monthlyOn(1, '09:00')->timezone('Europe/Madrid'); Schedule::command('stats:subscription-funnel')->weekly()->mondays()->at('09:15')->timezone('Europe/Madrid'); Schedule::command('stats:experiment-funnel')->weekly()->mondays()->at('09:30')->timezone('Europe/Madrid'); +Schedule::command('stats:price-experiment-funnel')->weekly()->mondays()->at('09:45')->timezone('Europe/Madrid'); diff --git a/tests/Feature/PriceExperimentTest.php b/tests/Feature/PriceExperimentTest.php new file mode 100644 index 00000000..30ffcb34 --- /dev/null +++ b/tests/Feature/PriceExperimentTest.php @@ -0,0 +1,169 @@ + true, + 'subscriptions.price_experiment.started_at' => '2026-06-01', + 'subscriptions.price_experiment.force_variant' => null, + 'subscriptions.price_experiment.variants.high' => [ + 'monthly' => ['price' => 8.99, 'original_price' => null, 'lookup' => 'whisper_pro_monthly_v9'], + 'yearly' => ['price' => 53.94, 'original_price' => 107.88, 'lookup' => 'whisper_pro_yearly_v9'], + ], + 'subscriptions.plans.monthly.price' => 3.99, + 'subscriptions.plans.monthly.stripe_lookup_key' => 'whisper_pro_monthly', + 'subscriptions.plans.yearly.price' => 23.88, + 'subscriptions.plans.yearly.stripe_lookup_key' => 'whisper_pro_yearly', + ]); +}); + +it('keeps users who registered before the experiment on the control price', function () { + $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-05-20')]); + + expect(app(ExperimentOffer::class)->priceVariantFor($user))->toBe(PriceExperiment::LEGACY); +}); + +it('treats a null (guest) scope as legacy', function () { + expect((new PriceExperiment)->resolve(null))->toBe(PriceExperiment::LEGACY); +}); + +it('treats everyone as legacy while the experiment is off', function () { + config(['subscriptions.price_experiment.started_at' => null]); + + $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); + + expect(app(ExperimentOffer::class)->priceVariantFor($user))->toBe(PriceExperiment::LEGACY); +}); + +it('pins every user to the forced winner price', function () { + config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]); + $offer = app(ExperimentOffer::class); + + $legacy = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-05-01')]); + $fresh = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); + + expect($offer->priceVariantFor($legacy))->toBe(PriceExperiment::HIGH) + ->and($offer->priceVariantFor($fresh))->toBe(PriceExperiment::HIGH); +}); + +it('ignores an invalid forced variant', function () { + config(['subscriptions.price_experiment.force_variant' => 'bogus']); + + $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-05-01')]); + + expect(app(ExperimentOffer::class)->priceVariantFor($user))->toBe(PriceExperiment::LEGACY); +}); + +it('splits post-start users across control and high and stays stable per user', function () { + $offer = app(ExperimentOffer::class); + $variants = []; + + for ($i = 0; $i < 40; $i++) { + $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); + $assigned = $offer->priceVariantFor($user); + $variants[] = $assigned; + + Feature::flushCache(); + expect($offer->priceVariantFor($user))->toBe($assigned); + } + + expect(array_unique($variants))->toEqualCanonicalizing([ + PriceExperiment::CONTROL, + PriceExperiment::HIGH, + ]); +}); + +it('assigns the price independently of the trial experiment (salted hash)', function () { + // For users the trial experiment buckets into pay_now, the price experiment + // must still produce BOTH price arms — otherwise the two are collinear. + $priceArmsWithinOneTrialArm = []; + + for ($i = 0; $i < 200 && count(array_unique($priceArmsWithinOneTrialArm)) < 2; $i++) { + $id = (string) Str::uuid(); + + if (SubscriptionExperiment::bucket($id) === SubscriptionExperiment::PAY_NOW) { + $priceArmsWithinOneTrialArm[] = PriceExperiment::bucket($id); + } + } + + expect(array_unique($priceArmsWithinOneTrialArm))->toEqualCanonicalizing([ + PriceExperiment::CONTROL, + PriceExperiment::HIGH, + ]); +}); + +it('applies the variant price and lookup key for a high-price user', function () { + $offer = app(ExperimentOffer::class); + $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); + Feature::for($user)->activate(PriceExperiment::class, PriceExperiment::HIGH); + + $plans = $offer->pricingPlansFor($user); + + expect($plans['monthly']['price'])->toBe(8.99) + ->and($plans['monthly']['stripe_lookup_key'])->toBe('whisper_pro_monthly_v9') + ->and($plans['yearly']['price'])->toBe(53.94) + ->and($plans['yearly']['original_price'])->toBe(107.88) + ->and($offer->lookupKeyFor($user, 'monthly'))->toBe('whisper_pro_monthly_v9') + ->and($offer->lookupKeyFor($user, 'yearly'))->toBe('whisper_pro_yearly_v9'); +}); + +it('leaves control-price users on the config prices', function () { + $offer = app(ExperimentOffer::class); + $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); + Feature::for($user)->activate(PriceExperiment::class, PriceExperiment::CONTROL); + + $plans = $offer->pricingPlansFor($user); + + expect($plans['monthly']['price'])->toBe(3.99) + ->and($plans['monthly']['stripe_lookup_key'])->toBe('whisper_pro_monthly') + ->and($offer->lookupKeyFor($user, 'monthly'))->toBe('whisper_pro_monthly'); +}); + +it('shows the assigned variant price on the paywall', function () { + $user = User::factory()->onboarded()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); + Feature::for($user)->activate(PriceExperiment::class, PriceExperiment::HIGH); + + $this->actingAs($user) + ->get(route('subscribe')) + ->assertOk() + ->assertInertia(fn ($page) => $page + ->component('subscription/paywall') + ->where('pricing.plans.monthly.price', 8.99) + ->where('pricing.plans.yearly.price', 53.94)); +}); + +it('charges the assigned variant price id at checkout, not a client-supplied one', function () { + config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]); + Cache::put('stripe_price_id:whisper_pro_monthly_v9', 'price_high_monthly', now()->addHour()); + + $checkout = Mockery::mock(Checkout::class); + $checkout->shouldReceive('toResponse')->andReturn(new RedirectResponse('https://stripe.test/session')); + + $builder = Mockery::mock(SubscriptionBuilder::class)->shouldIgnoreMissing(); + $builder->shouldReceive('checkout')->once()->andReturn($checkout); + + $user = Mockery::mock(User::class)->shouldIgnoreMissing(); + $user->shouldReceive('hasVerifiedEmail')->andReturn(true); + $user->shouldReceive('hasProPlan')->andReturn(false); + $user->shouldReceive('newSubscription') + ->once() + ->with('default', 'price_high_monthly') + ->andReturn($builder); + + $this->withoutMiddleware(HandleInertiaRequests::class); + $this->actingAs($user); + + $this->get(route('subscribe.checkout', ['plan' => 'monthly']))->assertRedirect(); +}); diff --git a/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php b/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php new file mode 100644 index 00000000..5be02025 --- /dev/null +++ b/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php @@ -0,0 +1,262 @@ + true, + 'subscriptions.price_experiment.started_at' => '2026-06-01', + 'subscriptions.price_experiment.force_variant' => null, + 'subscriptions.plans.monthly.trial_days' => 15, + 'ai_suggestions.report.excluded_emails' => [], + ]); + Carbon::setTestNow(CarbonImmutable::parse('2026-06-30 12:00:00')); + + // Seed the price→monthly-equivalent map so revenue is computed without Stripe: + // control €3.99, high €8.99. + Cache::put('experiment_funnel_monthly_equiv', ['price_control' => 399, 'price_high' => 899], now()->addHour()); +}); + +/** + * Create a user whose id buckets into the wanted price variant, with an optional + * default subscription, live (active) bank connections that drive the monthly cost, + * extra soft-deleted connections that count as "ever activated" but cost nothing, + * and optional AI consent. + * + * @param array{status: string, at: CarbonImmutable, price?: string, endsAt?: CarbonImmutable, trialEndsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription + */ +function priceUser(string $variant, CarbonImmutable $signup, ?array $subscription = null, int $connections = 0, int $deletedConnections = 0, bool $aiConsent = false): User +{ + do { + $id = (string) Str::uuid(); + } while (PriceExperiment::bucket($id) !== $variant); + + $user = User::factory()->create(['id' => $id, 'created_at' => $signup]); + + if ($subscription !== null) { + $user->subscriptions()->create([ + 'type' => 'default', + 'stripe_id' => 'sub_'.Str::random(12), + 'stripe_status' => $subscription['status'], + 'stripe_price' => $subscription['price'] ?? 'price_control', + 'created_at' => $subscription['at'], + 'ends_at' => $subscription['endsAt'] ?? null, + 'trial_ends_at' => $subscription['trialEndsAt'] ?? null, + 'refunded_at' => $subscription['refundedAt'] ?? null, + ]); + } + + if ($connections > 0) { + BankingConnection::factory()->count($connections)->for($user)->create(); + } + + if ($deletedConnections > 0) { + BankingConnection::factory()->count($deletedConnections)->for($user)->create() + ->each(fn (BankingConnection $connection) => $connection->delete()); + } + + if ($aiConsent) { + AiConsent::factory()->for($user)->create(); + } + + return $user; +} + +it('returns empty variants when the experiment has not started', function () { + config(['subscriptions.price_experiment.started_at' => null]); + + $report = app(PriceExperimentFunnelCollector::class)->collect(); + + expect($report['startedAt'])->toBeNull() + ->and($report['variants'][PriceExperiment::CONTROL]['assigned'])->toBe(0) + ->and($report['variants'][PriceExperiment::HIGH]['assigned'])->toBe(0); +}); + +it('attributes users to the right price variant and counts the funnel', function () { + $signup = CarbonImmutable::parse('2026-06-05'); // mature (15d + 3d) by the test clock + + priceUser(PriceExperiment::CONTROL, $signup); + priceUser(PriceExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay(), 'price' => 'price_control']); + priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup->addDay(), 'price' => 'price_high']); + + $variants = app(PriceExperimentFunnelCollector::class)->collect()['variants']; + + expect($variants[PriceExperiment::CONTROL]['assigned'])->toBe(2) + ->and($variants[PriceExperiment::CONTROL]['subscribed'])->toBe(1) + ->and($variants[PriceExperiment::CONTROL]['activeMature'])->toBe(1) + ->and($variants[PriceExperiment::CONTROL]['convertedMature'])->toBe(1) + ->and($variants[PriceExperiment::CONTROL]['conversionRate'])->toBe(0.5) + ->and($variants[PriceExperiment::HIGH]['assigned'])->toBe(1) + ->and($variants[PriceExperiment::HIGH]['activeMature'])->toBe(1); +}); + +it('books MRR at each variant price and reports contribution margin per assigned user', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + // High arm: one payer at €8.99 with 1 active connection (cost €0.40). + priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_high'], connections: 1); + // High arm: one assigned non-payer, no connections. + priceUser(PriceExperiment::HIGH, $signup); + + $high = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::HIGH]; + + expect($high['mrrCents'])->toBe(899) // €8.99 at the high price + ->and($high['costCents'])->toBe(40) // 1 active connection + ->and($high['contributionMarginCents'])->toBe(859) // 899 − 40 + ->and($high['assignedMature'])->toBe(2) + ->and($high['cmPerAssignedCents'])->toBe(430) // round(859 / 2) + ->and($high['connectionsPerPayer'])->toBe(1.0); +}); + +it('counts revoked connections as activation but charges cost only for live ones', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + // 1 live connection + 2 soft-deleted: activated (ever connected), but the + // monthly cost is only the 1 live connection — the stock-vs-flow fix. + priceUser(PriceExperiment::CONTROL, $signup, connections: 1, deletedConnections: 2); + + $control = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::CONTROL]; + + expect($control['activated'])->toBe(1) + ->and($control['activatedMature'])->toBe(1) + ->and($control['costCents'])->toBe(40); // 1 live × 40, not 3 × 40 +}); + +it('leaves young cohorts out of the matured metrics', function () { + priceUser(PriceExperiment::HIGH, CarbonImmutable::now()->subDays(5), [ + 'status' => 'active', 'at' => CarbonImmutable::now()->subDays(5), 'price' => 'price_high', + ]); + + $high = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::HIGH]; + + expect($high['assigned'])->toBe(1) + ->and($high['assignedMature'])->toBe(0) // 5d < 18d window + ->and($high['conversionRate'])->toBeNull(); +}); + +it('keeps the split even when a winner is forced, instead of collapsing onto one variant', function () { + config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]); + $signup = CarbonImmutable::parse('2026-06-05'); + + priceUser(PriceExperiment::CONTROL, $signup); + priceUser(PriceExperiment::HIGH, $signup); + + $variants = app(PriceExperimentFunnelCollector::class)->collect()['variants']; + + expect($variants[PriceExperiment::CONTROL]['assigned'])->toBe(1) + ->and($variants[PriceExperiment::HIGH]['assigned'])->toBe(1); +}); + +it('prints the primary CM/user (Welch) and conversion guardrail blocks but withholds the verdict at small n', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + for ($i = 0; $i < 3; $i++) { + priceUser(PriceExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_control']); + priceUser(PriceExperiment::CONTROL, $signup); + priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_high']); + priceUser(PriceExperiment::HIGH, $signup); + } + + Artisan::call('stats:price-experiment-funnel', ['--no-discord' => true]); + $output = Artisan::output(); + + expect($output)->toContain('PRIMARY') + ->and($output)->toContain('GUARDRAIL') + ->and($output)->toContain('Fisher exact') + ->and($output)->toContain('verdict withheld') // 6 per arm < MIN_WELCH_N + ->and($output)->not->toContain('α=0.05 ->'); // no significance verdict at small n +}); + +it('emits a Welch significance verdict once both arms clear the small-sample floor', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + // 30 per arm (== MIN_WELCH_N), half net-active payers, so there is real variance. + for ($i = 0; $i < 30; $i++) { + $sub = $i % 2 === 0 ? ['status' => 'active', 'at' => $signup] : null; + priceUser(PriceExperiment::CONTROL, $signup, $sub === null ? null : [...$sub, 'price' => 'price_control']); + priceUser(PriceExperiment::HIGH, $signup, $sub === null ? null : [...$sub, 'price' => 'price_high']); + } + + Artisan::call('stats:price-experiment-funnel', ['--no-discord' => true]); + $output = Artisan::output(); + + expect($output)->toContain('α=0.05 ->') + ->and($output)->not->toContain('verdict withheld'); +}); + +it('surfaces an unmapped-price warning in the report, not just the logs', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + // Net-active sub on a price id the seeded map does not know → MRR silently 0. + priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_rotated_old']); + + Artisan::call('stats:price-experiment-funnel', ['--no-discord' => true]); + $output = Artisan::output(); + + expect($output)->toContain('UNMAPPED PRICES') + ->and($output)->toContain('price_rotated_old'); +}); + +it('flags a sample-ratio mismatch when the assignment split is lopsided', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + for ($i = 0; $i < 10; $i++) { + priceUser(PriceExperiment::CONTROL, $signup); + } + priceUser(PriceExperiment::HIGH, $signup); + + artisan('stats:price-experiment-funnel', ['--no-discord' => true]) + ->expectsOutputToContain('SRM') + ->expectsOutputToContain('IMBALANCE') + ->assertSuccessful(); +}); + +it('does not flag SRM when the split is balanced', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + for ($i = 0; $i < 4; $i++) { + priceUser(PriceExperiment::CONTROL, $signup); + priceUser(PriceExperiment::HIGH, $signup); + } + + artisan('stats:price-experiment-funnel', ['--no-discord' => true]) + ->expectsOutputToContain('SRM') + ->doesntExpectOutputToContain('IMBALANCE') + ->assertSuccessful(); +}); + +it('posts the price experiment embed to discord with the monitoring-only warning', function () { + config(['services.discord.ai_cohort_webhook_url' => 'https://discord.test/hook']); + Http::fake(['discord.test/*' => Http::response('', 204)]); + + priceUser(PriceExperiment::CONTROL, CarbonImmutable::parse('2026-06-05'), [ + 'status' => 'active', 'at' => CarbonImmutable::parse('2026-06-05'), 'price' => 'price_control', + ]); + + artisan('stats:price-experiment-funnel')->assertSuccessful(); + + Http::assertSent(fn ($request) => $request->url() === 'https://discord.test/hook' + && str_contains($request['embeds'][0]['title'], 'Price Experiment') + && str_contains(json_encode($request['embeds'][0]), 'MONITORING ONLY')); +}); + +it('does not post when the experiment has not started', function () { + config(['subscriptions.price_experiment.started_at' => null]); + Http::fake(); + + artisan('stats:price-experiment-funnel')->assertSuccessful(); + + Http::assertNothingSent(); +}); diff --git a/tests/Feature/SyncStripePricesCommandTest.php b/tests/Feature/SyncStripePricesCommandTest.php index e9719792..cbd28603 100644 --- a/tests/Feature/SyncStripePricesCommandTest.php +++ b/tests/Feature/SyncStripePricesCommandTest.php @@ -68,6 +68,8 @@ beforeEach(function () { ], 'subscriptions.products.pro' => 'prod_test123', 'cashier.currency' => 'eur', + // Base-plan tests isolate the plans loop; variant syncing has its own test. + 'subscriptions.price_experiment.variants' => [], ]); }); @@ -149,3 +151,25 @@ test('returns success and warns when no plans are configured', function () { ->expectsOutputToContain('No plans found') ->assertSuccessful(); }); + +test('syncs the price-experiment variant tiers alongside the base plans', function () { + config([ + 'subscriptions.price_experiment.variants' => [ + 'high' => [ + 'monthly' => ['price' => 8.99, 'lookup' => 'whisper_pro_monthly_v9'], + 'yearly' => ['price' => 53.94, 'lookup' => 'whisper_pro_yearly_v9'], + ], + ], + ]); + + // Base plans already in sync; both high tiers are new → 2 created, 2 skipped. + bindMockStripeClientForSync([ + 'whisper_pro_monthly' => makeStripePrice('price_monthly', 399, 'eur', 'month'), + 'whisper_pro_yearly' => makeStripePrice('price_yearly', 2388, 'eur', 'year'), + ]); + + $this->artisan('stripe:sync-prices') + ->expectsOutputToContain('(price: high)') + ->expectsOutputToContain('2 created, 0 updated, 2 skipped') + ->assertSuccessful(); +}); diff --git a/tests/Unit/SampleRatioMismatchTest.php b/tests/Unit/SampleRatioMismatchTest.php new file mode 100644 index 00000000..0d0746e4 --- /dev/null +++ b/tests/Unit/SampleRatioMismatchTest.php @@ -0,0 +1,32 @@ +evenSplit(50, 50); + + expect($result['chiSq'])->toBe(0.0) + ->and($result['p'])->toEqualWithDelta(1.0, 1e-6); +}); + +it('returns a neutral result for empty arms', function () { + $result = (new SampleRatioMismatch)->evenSplit(0, 0); + + expect($result['chiSq'])->toBe(0.0) + ->and($result['p'])->toBe(1.0); +}); + +it('matches the known chi-square for a 60/40 split of 100', function () { + // χ² = ((60−50)² + (40−50)²)/50 = 4 → p = 2·(1 − Φ(2)) = 0.0455. + $result = (new SampleRatioMismatch)->evenSplit(60, 40); + + expect($result['chiSq'])->toEqualWithDelta(4.0, 1e-9) + ->and($result['p'])->toEqualWithDelta(0.0455, 1e-3); +}); + +it('flags a badly lopsided split with a tiny p', function () { + $result = (new SampleRatioMismatch)->evenSplit(90, 10); + + expect($result['chiSq'])->toEqualWithDelta(64.0, 1e-9) + ->and($result['p'])->toBeLessThan(0.001); +}); diff --git a/tests/Unit/WelchTTestTest.php b/tests/Unit/WelchTTestTest.php new file mode 100644 index 00000000..77b7d4e5 --- /dev/null +++ b/tests/Unit/WelchTTestTest.php @@ -0,0 +1,34 @@ +compare(10.0, 4.0, 100, 10.0, 4.0, 100); + + expect($result['diff'])->toBe(0.0) + ->and($result['t'])->toBe(0.0) + ->and($result['p'])->toEqualWithDelta(1.0, 1e-6); // erf approximation, |error| < 1.5e-7 +}); + +it('yields a two-sided p of ~0.317 when the difference equals one standard error', function () { + // varA/nA = varB/nB = 0.5 → seSq = 1, se = 1; diff = 1 → t = 1. + // Two-sided normal p = 2·(1 − Φ(1)) = 0.3173, validating the erf/normalCdf path. + $result = (new WelchTTest)->compare(1.0, 50.0, 100, 0.0, 50.0, 100); + + expect($result['t'])->toEqualWithDelta(1.0, 1e-9) + ->and($result['se'])->toEqualWithDelta(1.0, 1e-9) + ->and($result['p'])->toEqualWithDelta(0.3173, 1e-3); +}); + +it('finds a large separation highly significant', function () { + $result = (new WelchTTest)->compare(10.0, 4.0, 100, 8.0, 4.0, 100); + + expect($result['t'])->toEqualWithDelta(7.071, 1e-2) + ->and($result['p'])->toBeLessThan(0.001); +}); + +it('defers (p=1) when an arm has too few observations to estimate variance', function () { + $result = (new WelchTTest)->compare(5.0, 0.0, 1, 0.0, 0.0, 1); + + expect($result['p'])->toBe(1.0); +});