Merge 0e4501adbc into 2cca291773
This commit is contained in:
Víctor Falcón
GitHub
commit
7de05c8064
|
|
@ -0,0 +1,320 @@
|
|||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Features\PriceExperiment;
|
||||
use App\Services\Discord\DiscordWebhook;
|
||||
use App\Services\Stats\BinomialProportion;
|
||||
use App\Services\Stats\PriceExperimentFunnelCollector;
|
||||
use App\Services\Stats\ProportionSignificance;
|
||||
use App\Services\Stats\SampleRatioMismatch;
|
||||
use App\Services\Stats\WelchTTest;
|
||||
use App\Support\Money;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class SendPriceExperimentFunnelReportCommand extends Command
|
||||
{
|
||||
protected $signature = 'stats:price-experiment-funnel
|
||||
{--no-discord : Print the report to the console only, without posting to Discord}
|
||||
{--cost-per-connection=0.4 : Estimated monthly cost (in the Cashier currency) per active bank connection, used for the Cost/CM columns}';
|
||||
|
||||
protected $description = 'Post the price experiment funnel (control vs high) to Discord — MONITORING ONLY, decide at the pre-registered horizon';
|
||||
|
||||
private const LABELS = [
|
||||
PriceExperiment::CONTROL => 'control',
|
||||
PriceExperiment::HIGH => 'high',
|
||||
];
|
||||
|
||||
/**
|
||||
* Per-arm matured floor below which the Welch p-value (normal approximation to
|
||||
* Student's t) is too anti-conservative to trust, so the verdict is withheld.
|
||||
*/
|
||||
private const MIN_WELCH_N = 30;
|
||||
|
||||
public function __construct(
|
||||
private PriceExperimentFunnelCollector $collector,
|
||||
private ProportionSignificance $significance,
|
||||
private WelchTTest $welch,
|
||||
private SampleRatioMismatch $srm,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
public function handle(): int
|
||||
{
|
||||
$costPerConnectionCents = (int) round(((float) $this->option('cost-per-connection')) * 100);
|
||||
$report = $this->collector->collect($costPerConnectionCents);
|
||||
|
||||
if ($report['startedAt'] === null) {
|
||||
$this->warn('Price experiment not started — set PRICE_EXPERIMENT_STARTED_AT to begin.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
foreach ([...$this->tableLines($report), ...$this->integrityLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)] as $line) {
|
||||
$this->line($line);
|
||||
}
|
||||
|
||||
if ($this->option('no-discord')) {
|
||||
$this->info('Skipped Discord (--no-discord).');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$webhookUrl = config('services.discord.ai_cohort_webhook_url')
|
||||
?: config('services.discord.webhook_url');
|
||||
|
||||
(new DiscordWebhook($webhookUrl))->send('', [$this->buildEmbed($report)]);
|
||||
|
||||
$this->info('Price experiment funnel report sent to Discord.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{currency: string, revenueAvailable: bool, variants: array<string, array<string, mixed>>} $report
|
||||
* @return list<string>
|
||||
*/
|
||||
private function tableLines(array $report): array
|
||||
{
|
||||
$currency = $report['currency'];
|
||||
$lines = [sprintf(
|
||||
'%-8s %5s %5s %5s %5s %5s %6s %8s %8s %8s %8s %6s',
|
||||
'Variant', 'Assg', 'Actd', 'Card', 'MatU', 'Conv', 'Conv%', 'MRR', 'Cost', 'CM', 'CM/u', 'C/Pyr',
|
||||
)];
|
||||
|
||||
foreach (self::LABELS as $key => $label) {
|
||||
$row = $report['variants'][$key];
|
||||
$mature = $row['assignedMature'] > 0;
|
||||
$showMoney = $report['revenueAvailable'] && $mature;
|
||||
|
||||
$lines[] = sprintf(
|
||||
'%-8s %5d %5d %5d %5d %5d %6s %8s %8s %8s %8s %6s',
|
||||
$label,
|
||||
$row['assigned'],
|
||||
$row['activated'],
|
||||
$row['subscribed'],
|
||||
$row['assignedMature'],
|
||||
$row['convertedMature'],
|
||||
$mature ? ((int) round($row['conversionRate'] * 100)).'%' : 'pend',
|
||||
$showMoney ? Money::format($row['mrrCents'], $currency) : '—',
|
||||
$mature ? Money::format($row['costCents'], $currency) : '—',
|
||||
$showMoney ? Money::format($row['contributionMarginCents'], $currency) : '—',
|
||||
$showMoney && $row['cmPerAssignedCents'] !== null ? Money::format($row['cmPerAssignedCents'], $currency) : '—',
|
||||
$row['connectionsPerPayer'] !== null ? number_format($row['connectionsPerPayer'], 1) : '—',
|
||||
);
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Revenue-map integrity warnings, surfaced in the report the decision-maker
|
||||
* reads (not just the logs): Stripe unreachable, or net-active subscriptions on
|
||||
* price ids missing from the monthly-equivalent map — which would silently
|
||||
* undercount MRR/CM as 0 and bias the comparison against whichever arm's price
|
||||
* is unmapped. Empty (no lines) when everything resolves.
|
||||
*
|
||||
* @param array{revenueAvailable: bool, unmappedPriceIds: list<string>} $report
|
||||
* @return list<string>
|
||||
*/
|
||||
private function integrityLines(array $report): array
|
||||
{
|
||||
$lines = [];
|
||||
|
||||
if (! $report['revenueAvailable']) {
|
||||
$lines[] = '';
|
||||
$lines[] = '⚠ REVENUE UNAVAILABLE — Stripe prices could not be loaded; MRR/CM are blank this run.';
|
||||
}
|
||||
|
||||
if ($report['unmappedPriceIds'] !== []) {
|
||||
$lines[] = '';
|
||||
$lines[] = '⚠ UNMAPPED PRICES — net-active subs on price ids absent from the revenue map '
|
||||
.'(MRR undercounted as 0): '.implode(', ', $report['unmappedPriceIds'])
|
||||
.'. Confirm both arm prices are synced under the pro product (stripe:sync-prices).';
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample-ratio-mismatch check: the split must be ~50/50. A low p means the
|
||||
* assignment (or the pipeline that filters it) is broken, which invalidates
|
||||
* every comparison below — so this is checked first, on the raw assigned counts
|
||||
* and on the matured subset (to catch differential attrition).
|
||||
*
|
||||
* @param array{variants: array<string, array<string, mixed>>} $report
|
||||
* @return list<string>
|
||||
*/
|
||||
private function srmLines(array $report): array
|
||||
{
|
||||
$control = $report['variants'][PriceExperiment::CONTROL];
|
||||
$high = $report['variants'][PriceExperiment::HIGH];
|
||||
$lines = ['', 'SRM (assignment balance, expect 50/50 — a low p means the split is broken):'];
|
||||
|
||||
foreach (['assigned' => 'assigned', 'assignedMature' => 'matured'] as $field => $label) {
|
||||
$c = (int) $control[$field];
|
||||
$h = (int) $high[$field];
|
||||
|
||||
if ($c + $h === 0) {
|
||||
$lines[] = sprintf(' %-8s control %d / high %d (n=0)', $label, $c, $h);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$srm = $this->srm->evenSplit($c, $h);
|
||||
$flag = $srm['p'] < 0.01 ? ' ⚠ IMBALANCE — investigate before trusting results' : '';
|
||||
$lines[] = sprintf(' %-8s control %d / high %d χ²=%.2f p=%.3f%s', $label, $c, $h, $srm['chiSq'], $srm['p'], $flag);
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Primary decision test — Welch on contribution-margin-per-user (control vs
|
||||
* high), the metric we maximise — plus the conversion guardrail (Wilson CIs +
|
||||
* Fisher exact, a single control-vs-treatment contrast, so no multiplicity
|
||||
* correction). CM/user is the number to act on; conversion protects the funnel.
|
||||
*
|
||||
* @param array{revenueAvailable: bool, currency: string, variants: array<string, array<string, mixed>>} $report
|
||||
* @return list<string>
|
||||
*/
|
||||
private function significanceLines(array $report): array
|
||||
{
|
||||
$currency = $report['currency'];
|
||||
$control = $report['variants'][PriceExperiment::CONTROL];
|
||||
$high = $report['variants'][PriceExperiment::HIGH];
|
||||
|
||||
$lines = ['', 'PRIMARY — contribution margin per assigned user (Welch, high vs control):'];
|
||||
|
||||
if (! $report['revenueAvailable'] || $control['assignedMature'] < 2 || $high['assignedMature'] < 2) {
|
||||
$lines[] = ' pend — need mature volume in both arms and Stripe revenue to compare.';
|
||||
} else {
|
||||
foreach (['control' => $control, 'high' => $high] as $label => $row) {
|
||||
$lines[] = sprintf(' %-8s %8s/user (n=%d)', $label, Money::format((int) $row['cmPerAssignedCents'], $currency), $row['assignedMature']);
|
||||
}
|
||||
|
||||
$welch = $this->welch->compare(
|
||||
$this->cmMean($high), $this->cmVariance($high), $high['assignedMature'],
|
||||
$this->cmMean($control), $this->cmVariance($control), $control['assignedMature'],
|
||||
);
|
||||
$delta = Money::format((int) round($welch['diff']), $currency);
|
||||
$minN = min((int) $control['assignedMature'], (int) $high['assignedMature']);
|
||||
|
||||
if ($minN < self::MIN_WELCH_N) {
|
||||
// The p-value uses a normal approximation to Student's t; below a
|
||||
// per-arm floor its tails are too thin (anti-conservative), so we
|
||||
// show the effect size but withhold the verdict rather than risk a
|
||||
// small-sample false positive.
|
||||
$lines[] = sprintf(
|
||||
' Δ high−control: %s/user t=%.2f (small sample: min n=%d < %d → p unreliable, verdict withheld)',
|
||||
$delta, $welch['t'], $minN, self::MIN_WELCH_N,
|
||||
);
|
||||
} else {
|
||||
$significant = $welch['p'] < 0.05;
|
||||
$lines[] = sprintf(
|
||||
' Δ high−control: %s/user t=%.2f p=%.3f %s α=0.05 -> %s.%s',
|
||||
$delta, $welch['t'], $welch['p'], $significant ? '<' : '≥',
|
||||
$significant ? 'significant' : 'not significant',
|
||||
$significant ? '' : ' Keep running.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
$lines[] = '';
|
||||
$lines[] = 'GUARDRAIL — conversion (95% Wilson CI on Conv%, n = MatU):';
|
||||
$arms = [];
|
||||
|
||||
foreach (self::LABELS as $key => $label) {
|
||||
$row = $report['variants'][$key];
|
||||
$n = (int) $row['assignedMature'];
|
||||
$k = (int) $row['convertedMature'];
|
||||
|
||||
if ($n <= 0) {
|
||||
$lines[] = sprintf(' %-8s pend (n=0)', $label);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
[$lower, $upper] = $this->significance->wilsonInterval($k, $n);
|
||||
$lines[] = sprintf(' %-8s %6s [%6s – %6s] (n=%d)', $label, $this->percent($k / $n), $this->percent($lower), $this->percent($upper), $n);
|
||||
$arms[$key] = new BinomialProportion($label, $k, $n);
|
||||
}
|
||||
|
||||
if (count($arms) === 2) {
|
||||
$result = $this->significance->compare($arms[PriceExperiment::HIGH], $arms[PriceExperiment::CONTROL], comparisons: 1);
|
||||
$lines[] = sprintf(
|
||||
' Fisher exact p=%.3f %s α=%.3f -> %s.',
|
||||
$result['fisherP'], $result['significant'] ? '<' : '≥', $result['alpha'],
|
||||
$result['significant'] ? 'significant' : 'not significant',
|
||||
);
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $row
|
||||
*/
|
||||
private function cmMean(array $row): float
|
||||
{
|
||||
return $row['assignedMature'] > 0 ? (float) $row['contributionMarginCents'] / $row['assignedMature'] : 0.0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample variance (n−1) of per-user CM, from the accumulated moments.
|
||||
*
|
||||
* @param array<string, mixed> $row
|
||||
*/
|
||||
private function cmVariance(array $row): float
|
||||
{
|
||||
$n = (int) $row['assignedMature'];
|
||||
|
||||
if ($n < 2) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
$sum = (float) $row['contributionMarginCents'];
|
||||
|
||||
return max(0.0, ((float) $row['cmSumSq'] - ($sum * $sum) / $n) / ($n - 1));
|
||||
}
|
||||
|
||||
private function percent(float $rate): string
|
||||
{
|
||||
return number_format($rate * 100, 1).'%';
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{startedAt: CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, decisionWindowDays: int, unmappedPriceIds: list<string>, variants: array<string, array<string, mixed>>} $report
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function buildEmbed(array $report): array
|
||||
{
|
||||
return [
|
||||
'title' => '💶 Price Experiment — Funnel (control vs high)',
|
||||
'description' => "```\n".implode("\n", [...$this->tableLines($report), ...$this->integrityLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)])."\n```",
|
||||
'color' => 0x57F287,
|
||||
'fields' => [
|
||||
[
|
||||
'name' => 'Started',
|
||||
'value' => $report['startedAt']->format('D, d M Y').sprintf(' · new signups split 50/50; both arms mature at %d days.', $report['decisionWindowDays']),
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => 'Legend',
|
||||
'value' => sprintf(
|
||||
'Assg = signups · Actd = activated (bank or AI = cost triggered) · Card = started a subscription · MatU = matured assigned (old enough to score) · Conv%% = ever-charged net of refund ÷ MatU (time-invariant) · MRR = monthly run-rate of currently-paying subs · Cost = active connections × %s · CM = MRR − Cost · CM/u = CM ÷ MatU (the decision metric) · C/Pyr = active connections per payer.',
|
||||
Money::format($report['costPerConnectionCents'], $report['currency']),
|
||||
),
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => '⚠️ MONITORING ONLY',
|
||||
'value' => "**Do NOT call a winner from this weekly report.** Decide at the pre-registered horizon N (from the power analysis), not the first week Fisher/Welch crosses α — weekly peeking inflates the false-positive rate. The primary metric is **CM per assigned user** (Welch); conversion is a guardrail. CM/user is sub-cent per week at current volume, so early weeks read as 'not significant' — that is expected. Don't add/remove arms or change the split mid-flight. A 1-month CM cannot see price-driven churn, so re-read retention/LTV before permanently locking the price.",
|
||||
'inline' => false,
|
||||
],
|
||||
],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
|
@ -17,7 +17,7 @@ class SyncStripePricesCommand extends Command
|
|||
|
||||
public function handle(): int
|
||||
{
|
||||
$plans = config('subscriptions.plans', []);
|
||||
$plans = $this->plansToSync();
|
||||
$currency = config('cashier.currency', 'eur');
|
||||
$dryRun = $this->option('dry-run');
|
||||
|
||||
|
|
@ -103,6 +103,33 @@ class SyncStripePricesCommand extends Command
|
|||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* The base plans plus the price-experiment variant tiers, flattened into one
|
||||
* list of pseudo-plans so each gets its own Stripe price + lookup key. The
|
||||
* variant billing period and name are inherited from the matching base plan.
|
||||
*
|
||||
* @return array<string, array<string, mixed>>
|
||||
*/
|
||||
private function plansToSync(): array
|
||||
{
|
||||
$plans = (array) config('subscriptions.plans', []);
|
||||
|
||||
foreach ((array) config('subscriptions.price_experiment.variants', []) as $variantKey => $variantPlans) {
|
||||
foreach ((array) $variantPlans as $planKey => $spec) {
|
||||
$base = $plans[$planKey] ?? [];
|
||||
|
||||
$plans["price_experiment.{$variantKey}.{$planKey}"] = [
|
||||
'name' => ($base['name'] ?? ucfirst((string) $planKey))." (price: {$variantKey})",
|
||||
'price' => $spec['price'],
|
||||
'billing_period' => $base['billing_period'] ?? ($planKey === 'yearly' ? 'year' : 'month'),
|
||||
'stripe_lookup_key' => $spec['lookup'] ?? null,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return $plans;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Price|null
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
<?php
|
||||
|
||||
namespace App\Features;
|
||||
|
||||
use App\Models\User;
|
||||
use Carbon\CarbonImmutable;
|
||||
|
||||
/**
|
||||
* A/B assignment for the price experiment: control (the current plan prices) vs
|
||||
* a higher price tier. Independent of {@see SubscriptionExperiment} — the hash is
|
||||
* salted with a per-experiment prefix so a user's price bucket does not track
|
||||
* their trial bucket, keeping the two experiments statistically orthogonal and
|
||||
* leaving the salt free for future experiments on the same id.
|
||||
*
|
||||
* Users who registered before `subscriptions.price_experiment.started_at` (or any
|
||||
* user while it is null) are "legacy" and keep the control price. Everyone who
|
||||
* registered on or after the start is split evenly by a stable, salted hash of
|
||||
* their id, so the bucket never changes for a given user. The funnel report
|
||||
* mirrors bucket() in PHP so it always matches what the user was served.
|
||||
*
|
||||
* @api
|
||||
*/
|
||||
class PriceExperiment
|
||||
{
|
||||
public const LEGACY = 'legacy';
|
||||
|
||||
public const CONTROL = 'control';
|
||||
|
||||
public const HIGH = 'high';
|
||||
|
||||
/**
|
||||
* In-memory override that pins every user to the winning variant once the
|
||||
* experiment is decided. Returning non-null skips both storage and resolve,
|
||||
* so flipping PRICE_EXPERIMENT_FORCE_VARIANT rolls the winner out to everyone
|
||||
* without a deploy and without rewriting stored assignments.
|
||||
*/
|
||||
public function before(?User $user): ?string
|
||||
{
|
||||
$forced = config('subscriptions.price_experiment.force_variant');
|
||||
|
||||
return in_array($forced, [self::CONTROL, self::HIGH], true) ? $forced : null;
|
||||
}
|
||||
|
||||
public function resolve(?User $user): string
|
||||
{
|
||||
$startedAt = config('subscriptions.price_experiment.started_at');
|
||||
|
||||
if ($user === null || $startedAt === null || $user->created_at?->lt(CarbonImmutable::parse($startedAt))) {
|
||||
return self::LEGACY;
|
||||
}
|
||||
|
||||
return self::bucket((string) $user->getKey());
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic, evenly-split bucket for a post-start user. The 'price:' salt
|
||||
* decouples this split from every other crc32-based experiment on the same id
|
||||
* (e.g. SubscriptionExperiment), so the two arms are independent. Keep this the
|
||||
* single source of truth — the funnel report mirrors it to attribute users
|
||||
* without reading Pennant per row.
|
||||
*/
|
||||
public static function bucket(string $key): string
|
||||
{
|
||||
return crc32('price:'.$key) % 2 === 0 ? self::CONTROL : self::HIGH;
|
||||
}
|
||||
}
|
||||
|
|
@ -72,7 +72,10 @@ class SubscriptionController extends Controller
|
|||
abort(400, 'Invalid plan selected');
|
||||
}
|
||||
|
||||
$priceId = $this->resolvePriceIdByLookupKey($plan['stripe_lookup_key']);
|
||||
// The price tier is resolved server-side from the user's assigned variant,
|
||||
// never from the request, so a user can't pick the cheaper price.
|
||||
$lookupKey = $this->experimentOffer->lookupKeyFor($request->user(), $planKey);
|
||||
$priceId = $this->resolvePriceIdByLookupKey($lookupKey);
|
||||
|
||||
$subscriptionBuilder = $request->user()
|
||||
->newSubscription('default', $priceId);
|
||||
|
|
|
|||
|
|
@ -6,9 +6,11 @@ use App\Enums\BankingConnectionStatus;
|
|||
use App\Enums\BankingProvider;
|
||||
use App\Features\CalculateBalancesOnImport;
|
||||
use App\Features\Mcp;
|
||||
use App\Features\PriceExperiment;
|
||||
use App\Jobs\PurgeResidualEncryptionArtifactsJob;
|
||||
use App\Models\BankingConnection;
|
||||
use App\Services\CurrencyOptions;
|
||||
use App\Services\Subscriptions\ExperimentOffer;
|
||||
use Illuminate\Foundation\Inspiring;
|
||||
use Illuminate\Http\Request;
|
||||
use Inertia\Middleware;
|
||||
|
|
@ -16,7 +18,10 @@ use Laravel\Pennant\Feature;
|
|||
|
||||
class HandleInertiaRequests extends Middleware
|
||||
{
|
||||
public function __construct(private CurrencyOptions $currencyOptions) {}
|
||||
public function __construct(
|
||||
private CurrencyOptions $currencyOptions,
|
||||
private ExperimentOffer $experimentOffer,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* The root template that's loaded on the first page visit.
|
||||
|
|
@ -66,6 +71,18 @@ class HandleInertiaRequests extends Middleware
|
|||
PurgeResidualEncryptionArtifactsJob::dispatch($user);
|
||||
}
|
||||
|
||||
// Warm every Pennant feature this response reads in a single round-trip, so
|
||||
// the per-feature reads below (the price-experiment pricing variant and the
|
||||
// feature flags) hit Pennant's in-memory cache instead of each issuing its
|
||||
// own select + insert.
|
||||
if ($user) {
|
||||
Feature::for($user)->values([
|
||||
CalculateBalancesOnImport::class,
|
||||
Mcp::class,
|
||||
PriceExperiment::class,
|
||||
]);
|
||||
}
|
||||
|
||||
return [
|
||||
...parent::share($request),
|
||||
'flash' => [
|
||||
|
|
@ -95,7 +112,9 @@ class HandleInertiaRequests extends Middleware
|
|||
'subscriptionsEnabled' => config('subscriptions.enabled', false),
|
||||
'aiCategorizationUpsellRate' => (int) config('ai_categorization.upsell_sample_rate'),
|
||||
'pricing' => [
|
||||
'plans' => config('subscriptions.plans', []),
|
||||
'plans' => $user
|
||||
? $this->experimentOffer->pricingPlansFor($user)
|
||||
: config('subscriptions.plans', []),
|
||||
'defaultPlan' => config('subscriptions.default_plan', 'monthly'),
|
||||
'bestValuePlan' => config('subscriptions.best_value_plan', null),
|
||||
'promo' => config('subscriptions.promo', []),
|
||||
|
|
|
|||
|
|
@ -5,9 +5,7 @@ namespace App\Services\Stats;
|
|||
use App\Features\SubscriptionExperiment;
|
||||
use App\Models\User;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Laravel\Cashier\Cashier;
|
||||
use Laravel\Cashier\Subscription;
|
||||
|
||||
class ExperimentFunnelCollector
|
||||
|
|
@ -18,6 +16,8 @@ class ExperimentFunnelCollector
|
|||
*/
|
||||
private const SETTLE_BUFFER_DAYS = 3;
|
||||
|
||||
public function __construct(private MonthlyEquivalentPrices $prices) {}
|
||||
|
||||
/**
|
||||
* Per-variant funnel for the trial/pricing experiment. Users are attributed
|
||||
* by SubscriptionExperiment::bucket() — the deterministic crc32 split that is
|
||||
|
|
@ -94,7 +94,7 @@ class ExperimentFunnelCollector
|
|||
$now = CarbonImmutable::now('UTC');
|
||||
$excluded = (array) config('ai_suggestions.report.excluded_emails', []);
|
||||
$windows = $this->decisionWindows();
|
||||
$monthlyEquiv = $this->monthlyEquivByPriceId();
|
||||
$monthlyEquiv = $this->prices->map();
|
||||
$missingPrices = [];
|
||||
|
||||
User::query()
|
||||
|
|
@ -270,70 +270,6 @@ class ExperimentFunnelCollector
|
|||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Monthly-equivalent amount (in cents) for each plan price id, from Stripe.
|
||||
* Yearly prices are divided by 12. Fetched by product so that archived,
|
||||
* rotated price ids (Stripe mints a new id and transfers the lookup key on
|
||||
* any amount change) still resolve — otherwise subscriptions on an old id
|
||||
* would silently contribute 0 to MRR. Falls back to the current lookup keys
|
||||
* when no product is configured. Foreign-currency and one-off prices are
|
||||
* skipped. Cached for an hour; returns [] (revenue unavailable) if Stripe
|
||||
* can't be reached, without caching the failure.
|
||||
*
|
||||
* @return array<string, int>
|
||||
*/
|
||||
private function monthlyEquivByPriceId(): array
|
||||
{
|
||||
$key = 'experiment_funnel_monthly_equiv';
|
||||
|
||||
if (Cache::has($key)) {
|
||||
return Cache::get($key);
|
||||
}
|
||||
|
||||
$productId = config('subscriptions.products.pro');
|
||||
$lookups = array_values(array_filter([
|
||||
config('subscriptions.plans.monthly.stripe_lookup_key'),
|
||||
config('subscriptions.plans.yearly.stripe_lookup_key'),
|
||||
]));
|
||||
|
||||
if ($productId === null && $lookups === []) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$params = $productId !== null
|
||||
? ['product' => $productId, 'limit' => 100]
|
||||
: ['lookup_keys' => $lookups, 'limit' => 10];
|
||||
|
||||
try {
|
||||
$prices = Cashier::stripe()->prices->all($params);
|
||||
} catch (\Throwable) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$currency = strtolower((string) config('cashier.currency', 'eur'));
|
||||
$map = [];
|
||||
foreach ($prices->data as $price) {
|
||||
if ($price->recurring === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (strtolower((string) ($price->currency ?? $currency)) !== $currency) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$amount = (int) ($price->unit_amount ?? 0);
|
||||
$map[$price->id] = ($price->recurring->interval ?? 'month') === 'year'
|
||||
? (int) round($amount / 12)
|
||||
: $amount;
|
||||
}
|
||||
|
||||
if ($map !== []) {
|
||||
Cache::put($key, $map, now()->addHour());
|
||||
}
|
||||
|
||||
return $map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Days from signup until each variant's outcome can be scored: the trial
|
||||
* length (the longer of the two reduced trials) or the refund window.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Stats;
|
||||
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Laravel\Cashier\Cashier;
|
||||
|
||||
/**
|
||||
* Monthly-equivalent amount (in cents) for each active recurring Stripe price id
|
||||
* under the pro product, yearly prices divided by 12. Fetched by product so that
|
||||
* archived, rotated price ids (Stripe mints a new id and transfers the lookup key
|
||||
* on any amount change) still resolve — otherwise subscriptions on an old id would
|
||||
* silently contribute 0 to MRR — and so every experiment price tier is covered by
|
||||
* one call. Falls back to the current lookup keys when no product is configured.
|
||||
* Foreign-currency and one-off prices are skipped. Cached for an hour; returns []
|
||||
* (revenue unavailable) if Stripe can't be reached, without caching the failure.
|
||||
*
|
||||
* Shared by the trial and price experiment funnel collectors so "what does each
|
||||
* price cost per month" has a single source of truth.
|
||||
*/
|
||||
class MonthlyEquivalentPrices
|
||||
{
|
||||
private const CACHE_KEY = 'experiment_funnel_monthly_equiv';
|
||||
|
||||
/**
|
||||
* @return array<string, int>
|
||||
*/
|
||||
public function map(): array
|
||||
{
|
||||
if (Cache::has(self::CACHE_KEY)) {
|
||||
return Cache::get(self::CACHE_KEY);
|
||||
}
|
||||
|
||||
$productId = config('subscriptions.products.pro');
|
||||
$lookups = array_values(array_filter([
|
||||
config('subscriptions.plans.monthly.stripe_lookup_key'),
|
||||
config('subscriptions.plans.yearly.stripe_lookup_key'),
|
||||
]));
|
||||
|
||||
if ($productId === null && $lookups === []) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$params = $productId !== null
|
||||
? ['product' => $productId, 'limit' => 100]
|
||||
: ['lookup_keys' => $lookups, 'limit' => 10];
|
||||
|
||||
try {
|
||||
$prices = Cashier::stripe()->prices->all($params);
|
||||
} catch (\Throwable) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$currency = strtolower((string) config('cashier.currency', 'eur'));
|
||||
$map = [];
|
||||
foreach ($prices->data as $price) {
|
||||
if ($price->recurring === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (strtolower((string) ($price->currency ?? $currency)) !== $currency) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$amount = (int) ($price->unit_amount ?? 0);
|
||||
$map[$price->id] = ($price->recurring->interval ?? 'month') === 'year'
|
||||
? (int) round($amount / 12)
|
||||
: $amount;
|
||||
}
|
||||
|
||||
if ($map !== []) {
|
||||
Cache::put(self::CACHE_KEY, $map, now()->addHour());
|
||||
}
|
||||
|
||||
return $map;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Stats;
|
||||
|
||||
/**
|
||||
* Standard-normal cumulative distribution, shared by the significance tests that
|
||||
* need a normal tail probability (Welch's t via the large-sample approximation,
|
||||
* and the χ²₁ sample-ratio-mismatch check). One implementation so the erf
|
||||
* approximation lives in a single place.
|
||||
*/
|
||||
final class Normal
|
||||
{
|
||||
public static function cdf(float $x): float
|
||||
{
|
||||
return 0.5 * (1.0 + self::erf($x / sqrt(2.0)));
|
||||
}
|
||||
|
||||
/** Abramowitz & Stegun 7.1.26 — |error| < 1.5e-7. */
|
||||
private static function erf(float $x): float
|
||||
{
|
||||
$sign = $x < 0 ? -1.0 : 1.0;
|
||||
$x = abs($x);
|
||||
$t = 1.0 / (1.0 + 0.3275911 * $x);
|
||||
$y = 1.0 - ((((1.061405429 * $t - 1.453152027) * $t + 1.421413741) * $t - 0.284496736) * $t + 0.254829592) * $t * exp(-$x * $x);
|
||||
|
||||
return $sign * $y;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,243 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Stats;
|
||||
|
||||
use App\Features\PriceExperiment;
|
||||
use App\Models\User;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Laravel\Cashier\Subscription;
|
||||
|
||||
/**
|
||||
* Per-variant funnel for the price experiment (control vs high). Users are
|
||||
* attributed by PriceExperiment::bucket() — the deterministic, salted crc32 split
|
||||
* that is the single source of truth for assignment — so it matches the price each
|
||||
* user was served without being perturbed by the force_variant rollout hook.
|
||||
*
|
||||
* Both arms share ONE decision window (the trial length + a settle buffer), so the
|
||||
* matured cohorts are directly comparable — unlike the trial experiment, where each
|
||||
* variant matured on its own window. The funnel is
|
||||
* assigned → activated → subscribed → net-paying.
|
||||
*
|
||||
* The decision metric is contribution margin per assigned user (MRR − connection
|
||||
* cost, ÷ assigned). Cost is the *currently-active* connections × the flat
|
||||
* per-connection estimate, matching the monthly MRR flow (revoked connections cost
|
||||
* nothing going forward). Per-user CM is accumulated as moments (sum via
|
||||
* contributionMarginCents, sum-of-squares via cmSumSq) so the report can run a
|
||||
* Welch test on CM/user across arms without retaining every value. Conversion —
|
||||
* ever-charged net of refund, time-invariant — is the guardrail metric.
|
||||
*
|
||||
* ponytail: connection cost is a blended flat estimate across all providers (only
|
||||
* Enable Banking actually bills us; API-key providers are free). It is symmetric
|
||||
* across arms, so it biases absolute CM but not the between-arm comparison. Make it
|
||||
* per-provider only if absolute margin ever needs to be trusted.
|
||||
*/
|
||||
class PriceExperimentFunnelCollector
|
||||
{
|
||||
/** Days after the trial ends before a user's outcome is settled enough to score. */
|
||||
private const SETTLE_BUFFER_DAYS = 3;
|
||||
|
||||
public function __construct(private MonthlyEquivalentPrices $prices) {}
|
||||
|
||||
/**
|
||||
* @param int $costPerConnectionCents flat estimated monthly cost per active connection
|
||||
* @return array{
|
||||
* startedAt: ?CarbonImmutable,
|
||||
* currency: string,
|
||||
* revenueAvailable: bool,
|
||||
* costPerConnectionCents: int,
|
||||
* decisionWindowDays: int,
|
||||
* unmappedPriceIds: list<string>,
|
||||
* variants: array<string, array{
|
||||
* assigned: int, activated: int, subscribed: int,
|
||||
* assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int,
|
||||
* payerCount: int, payerConnectionSum: int,
|
||||
* conversionRate: ?float,
|
||||
* mrrCents: int, costCents: int, contributionMarginCents: int,
|
||||
* cmPerAssignedCents: ?int, cmSumSq: float, connectionsPerPayer: ?float,
|
||||
* }>
|
||||
* }
|
||||
*/
|
||||
public function collect(int $costPerConnectionCents = 40): array
|
||||
{
|
||||
$startedValue = config('subscriptions.price_experiment.started_at');
|
||||
$startedAt = $startedValue !== null ? CarbonImmutable::parse($startedValue) : null;
|
||||
$currency = strtoupper((string) config('cashier.currency', 'eur'));
|
||||
$window = (int) config('subscriptions.plans.monthly.trial_days', 15) + self::SETTLE_BUFFER_DAYS;
|
||||
|
||||
$variants = [
|
||||
PriceExperiment::CONTROL => $this->emptyRow(),
|
||||
PriceExperiment::HIGH => $this->emptyRow(),
|
||||
];
|
||||
|
||||
if ($startedAt === null) {
|
||||
return [
|
||||
'startedAt' => null,
|
||||
'currency' => $currency,
|
||||
'revenueAvailable' => false,
|
||||
'costPerConnectionCents' => $costPerConnectionCents,
|
||||
'decisionWindowDays' => $window,
|
||||
'unmappedPriceIds' => [],
|
||||
'variants' => $variants,
|
||||
];
|
||||
}
|
||||
|
||||
$now = CarbonImmutable::now('UTC');
|
||||
$excluded = (array) config('ai_suggestions.report.excluded_emails', []);
|
||||
$monthlyEquiv = $this->prices->map();
|
||||
$missingPrices = [];
|
||||
|
||||
User::query()
|
||||
->withTrashed()
|
||||
->where('users.created_at', '>=', $startedAt)
|
||||
->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded))
|
||||
->with(['subscriptions' => fn ($query) => $query->where('type', 'default')])
|
||||
->select(['id', 'created_at'])
|
||||
->withCount([
|
||||
// "Ever connected" drives activation (cost was triggered at some point).
|
||||
'bankingConnections as connection_count' => fn ($query) => $query->withTrashed(),
|
||||
// "Currently active" drives the monthly recurring cost.
|
||||
'bankingConnections as active_connection_count',
|
||||
'aiConsents as ai_consent_count',
|
||||
])
|
||||
->chunkById(500, function ($users) use (&$variants, &$missingPrices, $window, $now, $monthlyEquiv, $costPerConnectionCents): void {
|
||||
foreach ($users as $user) {
|
||||
$variant = PriceExperiment::bucket((string) $user->id);
|
||||
|
||||
if (! isset($variants[$variant])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$row = &$variants[$variant];
|
||||
|
||||
$row['assigned']++;
|
||||
|
||||
$everConnections = (int) ($user->connection_count ?? 0);
|
||||
$activeConnections = (int) ($user->active_connection_count ?? 0);
|
||||
$activated = $everConnections > 0 || (int) ($user->ai_consent_count ?? 0) > 0;
|
||||
|
||||
if ($activated) {
|
||||
$row['activated']++;
|
||||
}
|
||||
|
||||
/** @var Subscription|null $subscription */
|
||||
$subscription = $user->subscriptions->sortByDesc('created_at')->first();
|
||||
$status = $subscription?->stripe_status;
|
||||
$netActive = $status === 'active' && $subscription->refunded_at === null;
|
||||
|
||||
// Time-invariant "ever charged, net of refund": comparable across
|
||||
// cohorts of different ages. Excludes trial-only cancels.
|
||||
$converted = $subscription !== null
|
||||
&& $subscription->refunded_at === null
|
||||
&& $status !== 'trialing'
|
||||
&& (
|
||||
$subscription->trial_ends_at === null
|
||||
|| $subscription->ends_at === null
|
||||
|| $subscription->ends_at->greaterThan($subscription->trial_ends_at)
|
||||
);
|
||||
|
||||
if ($subscription !== null) {
|
||||
$row['subscribed']++;
|
||||
}
|
||||
|
||||
$mature = CarbonImmutable::parse($user->created_at)
|
||||
->addDays($window)
|
||||
->lessThanOrEqualTo($now);
|
||||
|
||||
if (! $mature) {
|
||||
unset($row);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$row['assignedMature']++;
|
||||
|
||||
if ($activated) {
|
||||
$row['activatedMature']++;
|
||||
}
|
||||
|
||||
if ($converted) {
|
||||
$row['convertedMature']++;
|
||||
}
|
||||
|
||||
$costCents = $activeConnections * $costPerConnectionCents;
|
||||
$row['costCents'] += $costCents;
|
||||
|
||||
$revenueCents = 0;
|
||||
if ($netActive) {
|
||||
$row['activeMature']++;
|
||||
$priceId = (string) $subscription->stripe_price;
|
||||
|
||||
if ($monthlyEquiv !== [] && ! isset($monthlyEquiv[$priceId])) {
|
||||
$missingPrices[$priceId] = true;
|
||||
}
|
||||
|
||||
$revenueCents = (int) ($monthlyEquiv[$priceId] ?? 0);
|
||||
$row['mrrCents'] += $revenueCents;
|
||||
$row['payerCount']++;
|
||||
$row['payerConnectionSum'] += $activeConnections;
|
||||
}
|
||||
|
||||
// Per-user monthly CM; cmSumSq feeds the Welch variance. The sum
|
||||
// equals mrrCents − costCents, so it is not stored separately.
|
||||
$cmUser = $revenueCents - $costCents;
|
||||
$row['cmSumSq'] += $cmUser * $cmUser;
|
||||
|
||||
unset($row);
|
||||
}
|
||||
});
|
||||
|
||||
if ($missingPrices !== []) {
|
||||
Log::warning('Price experiment funnel: net-active subscriptions on prices absent from the monthly-equivalent map — their MRR is undercounted as 0.', [
|
||||
'price_ids' => array_keys($missingPrices),
|
||||
]);
|
||||
}
|
||||
|
||||
foreach ($variants as $key => $row) {
|
||||
$n = $row['assignedMature'];
|
||||
$variants[$key]['conversionRate'] = $n > 0 ? (float) $row['convertedMature'] / $n : null;
|
||||
$variants[$key]['contributionMarginCents'] = $row['mrrCents'] - $row['costCents'];
|
||||
$variants[$key]['cmPerAssignedCents'] = $n > 0
|
||||
? (int) round(($row['mrrCents'] - $row['costCents']) / $n)
|
||||
: null;
|
||||
$variants[$key]['connectionsPerPayer'] = $row['payerCount'] > 0
|
||||
? (float) $row['payerConnectionSum'] / $row['payerCount']
|
||||
: null;
|
||||
}
|
||||
|
||||
return [
|
||||
'startedAt' => $startedAt,
|
||||
'currency' => $currency,
|
||||
'revenueAvailable' => $monthlyEquiv !== [],
|
||||
'costPerConnectionCents' => $costPerConnectionCents,
|
||||
'decisionWindowDays' => $window,
|
||||
'unmappedPriceIds' => array_keys($missingPrices),
|
||||
'variants' => $variants,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{assigned: int, activated: int, subscribed: int, assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int, payerCount: int, payerConnectionSum: int, conversionRate: ?float, mrrCents: int, costCents: int, contributionMarginCents: int, cmPerAssignedCents: ?int, cmSumSq: float, connectionsPerPayer: ?float}
|
||||
*/
|
||||
private function emptyRow(): array
|
||||
{
|
||||
return [
|
||||
'assigned' => 0,
|
||||
'activated' => 0,
|
||||
'subscribed' => 0,
|
||||
'assignedMature' => 0,
|
||||
'activatedMature' => 0,
|
||||
'convertedMature' => 0,
|
||||
'activeMature' => 0,
|
||||
'payerCount' => 0,
|
||||
'payerConnectionSum' => 0,
|
||||
'conversionRate' => null,
|
||||
'mrrCents' => 0,
|
||||
'costCents' => 0,
|
||||
'contributionMarginCents' => 0,
|
||||
'cmPerAssignedCents' => null,
|
||||
'cmSumSq' => 0.0,
|
||||
'connectionsPerPayer' => null,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Stats;
|
||||
|
||||
/**
|
||||
* Sample-ratio-mismatch (SRM) check for a two-arm experiment: a χ² goodness-of-fit
|
||||
* test (1 degree of freedom) of the observed arm counts against an even 1:1 split.
|
||||
* A low p means the assignment is not really 50/50 — a data-pipeline or attrition
|
||||
* bug that invalidates every downstream comparison, so it must be checked before
|
||||
* trusting the funnel. Run it on the assigned counts AND on any filtered subset
|
||||
* (matured / activated) to catch differential attrition a deterministic assignment
|
||||
* can still produce.
|
||||
*/
|
||||
final class SampleRatioMismatch
|
||||
{
|
||||
/**
|
||||
* @return array{chiSq: float, p: float}
|
||||
*/
|
||||
public function evenSplit(int $countA, int $countB): array
|
||||
{
|
||||
$total = $countA + $countB;
|
||||
|
||||
if ($total === 0) {
|
||||
return ['chiSq' => 0.0, 'p' => 1.0];
|
||||
}
|
||||
|
||||
$expected = $total / 2.0;
|
||||
$chiSq = (($countA - $expected) ** 2 + ($countB - $expected) ** 2) / $expected;
|
||||
|
||||
// χ² with 1 df is Z²; its upper tail is P(|Z| > √χ²).
|
||||
$p = 2.0 * (1.0 - Normal::cdf(sqrt($chiSq)));
|
||||
|
||||
return ['chiSq' => $chiSq, 'p' => $p];
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Stats;
|
||||
|
||||
/**
|
||||
* Welch's two-sample test for a difference in means with unequal variances — the
|
||||
* correct test for comparing contribution-margin-per-user across price arms. That
|
||||
* metric is heavily zero-inflated and, because a higher-priced arm books several
|
||||
* times the revenue variance of the control at equal conversion, a pooled-variance
|
||||
* t-test is invalid here. The two-sided p-value uses the normal approximation to
|
||||
* Student's t, accurate at the per-arm sample sizes this experiment reaches
|
||||
* (n ≳ 100 by the central limit theorem); the Satterthwaite degrees of freedom are
|
||||
* returned for context.
|
||||
*
|
||||
* ponytail: normal-approx p-value, exact only for large n. If a future experiment
|
||||
* needs it at small n, swap Normal::cdf() for a Student-t CDF (regularised
|
||||
* incomplete beta) — the statistic and df here are already exact.
|
||||
*/
|
||||
final class WelchTTest
|
||||
{
|
||||
/**
|
||||
* @param float $varA sample variance (n−1 denominator)
|
||||
* @param float $varB sample variance (n−1 denominator)
|
||||
* @return array{diff: float, se: float, t: float, df: float, p: float}
|
||||
*/
|
||||
public function compare(float $meanA, float $varA, int $nA, float $meanB, float $varB, int $nB): array
|
||||
{
|
||||
$seSq = ($nA > 0 ? $varA / $nA : 0.0) + ($nB > 0 ? $varB / $nB : 0.0);
|
||||
$se = sqrt($seSq);
|
||||
$diff = $meanA - $meanB;
|
||||
|
||||
if ($se <= 0.0 || $nA < 2 || $nB < 2) {
|
||||
return ['diff' => $diff, 'se' => $se, 't' => 0.0, 'df' => 0.0, 'p' => 1.0];
|
||||
}
|
||||
|
||||
$t = $diff / $se;
|
||||
$df = ($seSq ** 2) / (
|
||||
(($varA / $nA) ** 2) / ($nA - 1)
|
||||
+ (($varB / $nB) ** 2) / ($nB - 1)
|
||||
);
|
||||
$p = 2.0 * (1.0 - Normal::cdf(abs($t)));
|
||||
|
||||
return ['diff' => $diff, 'se' => $se, 't' => $t, 'df' => $df, 'p' => $p];
|
||||
}
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
namespace App\Services\Subscriptions;
|
||||
|
||||
use App\Features\PriceExperiment;
|
||||
use App\Features\SubscriptionExperiment;
|
||||
use App\Models\User;
|
||||
use Carbon\CarbonImmutable;
|
||||
|
|
@ -9,10 +10,11 @@ use Laravel\Cashier\Subscription;
|
|||
use Laravel\Pennant\Feature;
|
||||
|
||||
/**
|
||||
* Translates a user's experiment variant into the concrete offer they get:
|
||||
* how many trial days per plan, whether they pay immediately, and whether they
|
||||
* can still trigger the self-service refund. Single source of truth shared by
|
||||
* the checkout, the paywall and the billing settings screen.
|
||||
* Translates a user's experiment variants into the concrete offer they get: the
|
||||
* plan prices (price experiment), how many trial days per plan, whether they pay
|
||||
* immediately, and whether they can still trigger the self-service refund. Single
|
||||
* source of truth shared by the checkout, the paywall and the billing settings
|
||||
* screen, so a user is always shown exactly the price and terms they are charged.
|
||||
*/
|
||||
class ExperimentOffer
|
||||
{
|
||||
|
|
@ -21,6 +23,49 @@ class ExperimentOffer
|
|||
return Feature::for($user)->value(SubscriptionExperiment::class);
|
||||
}
|
||||
|
||||
public function priceVariantFor(User $user): string
|
||||
{
|
||||
return Feature::for($user)->value(PriceExperiment::class);
|
||||
}
|
||||
|
||||
/**
|
||||
* The plans config with the user's price-experiment variant applied: price,
|
||||
* original_price and Stripe lookup key swapped for the assigned tier. Control
|
||||
* and legacy users get the unchanged config. Drives both the paywall display
|
||||
* and the checkout charge, so the shown price always equals the charged one.
|
||||
*
|
||||
* @return array<string, array<string, mixed>>
|
||||
*/
|
||||
public function pricingPlansFor(User $user): array
|
||||
{
|
||||
$plans = (array) config('subscriptions.plans', []);
|
||||
$overrides = (array) config('subscriptions.price_experiment.variants.'.$this->priceVariantFor($user), []);
|
||||
|
||||
foreach ($overrides as $planKey => $override) {
|
||||
if (! isset($plans[$planKey])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$plans[$planKey]['price'] = $override['price'];
|
||||
$plans[$planKey]['original_price'] = $override['original_price'] ?? null;
|
||||
$plans[$planKey]['stripe_lookup_key'] = $override['lookup'];
|
||||
}
|
||||
|
||||
return $plans;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stripe lookup key to charge for the given plan and the user's price variant.
|
||||
* Falls back to the control plan's key for control/legacy users.
|
||||
*/
|
||||
public function lookupKeyFor(User $user, string $planKey): string
|
||||
{
|
||||
return (string) config(
|
||||
"subscriptions.price_experiment.variants.{$this->priceVariantFor($user)}.{$planKey}.lookup",
|
||||
config("subscriptions.plans.{$planKey}.stripe_lookup_key"),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Trial days to apply at checkout for the given plan and the user's variant.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -45,6 +45,43 @@ return [
|
|||
'pay_now_refund_window_days' => (int) env('SUBSCRIPTION_EXPERIMENT_REFUND_WINDOW_DAYS', 3),
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Price Experiment
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| A/B test on the *price* of the paid plan, independent of the trial
|
||||
| experiment above. Users who register on or after `started_at` are split
|
||||
| evenly into `control` (the plans.* prices) and `high` (the variant prices
|
||||
| below); earlier users stay "legacy" and keep the control price. While
|
||||
| `started_at` is null the experiment is off and everyone gets control, so
|
||||
| this is a no-op until activated. Each variant needs its own Stripe prices —
|
||||
| run `php artisan stripe:sync-prices` before setting `started_at`. The yearly
|
||||
| price is the monthly × 6, matching the control plan structure.
|
||||
|
|
||||
*/
|
||||
|
||||
'price_experiment' => [
|
||||
'started_at' => env('PRICE_EXPERIMENT_STARTED_AT'),
|
||||
// Once a winner is chosen, set this to control / high to give every user
|
||||
// that price and end the split (env-only, no deploy).
|
||||
'force_variant' => env('PRICE_EXPERIMENT_FORCE_VARIANT'),
|
||||
'variants' => [
|
||||
'high' => [
|
||||
'monthly' => [
|
||||
'price' => (float) env('PRICE_EXPERIMENT_HIGH_MONTHLY', 8.99),
|
||||
'original_price' => null,
|
||||
'lookup' => env('PRICE_EXPERIMENT_HIGH_MONTHLY_LOOKUP_KEY', 'whisper_pro_monthly_v9'),
|
||||
],
|
||||
'yearly' => [
|
||||
'price' => (float) env('PRICE_EXPERIMENT_HIGH_YEARLY', 53.94),
|
||||
'original_price' => (float) env('PRICE_EXPERIMENT_HIGH_YEARLY_ORIGINAL', 107.88),
|
||||
'lookup' => env('PRICE_EXPERIMENT_HIGH_YEARLY_LOOKUP_KEY', 'whisper_pro_yearly_v9'),
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Stripe Product IDs
|
||||
|
|
|
|||
|
|
@ -17,3 +17,4 @@ Schedule::command('stats:daily-report')->dailyAt('09:00')->timezone('Europe/Madr
|
|||
Schedule::command('stats:ai-cohort-report')->monthlyOn(1, '09:00')->timezone('Europe/Madrid');
|
||||
Schedule::command('stats:subscription-funnel')->weekly()->mondays()->at('09:15')->timezone('Europe/Madrid');
|
||||
Schedule::command('stats:experiment-funnel')->weekly()->mondays()->at('09:30')->timezone('Europe/Madrid');
|
||||
Schedule::command('stats:price-experiment-funnel')->weekly()->mondays()->at('09:45')->timezone('Europe/Madrid');
|
||||
|
|
|
|||
|
|
@ -0,0 +1,169 @@
|
|||
<?php
|
||||
|
||||
use App\Features\PriceExperiment;
|
||||
use App\Features\SubscriptionExperiment;
|
||||
use App\Http\Middleware\HandleInertiaRequests;
|
||||
use App\Models\User;
|
||||
use App\Services\Subscriptions\ExperimentOffer;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Http\RedirectResponse;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Str;
|
||||
use Laravel\Cashier\Checkout;
|
||||
use Laravel\Cashier\SubscriptionBuilder;
|
||||
use Laravel\Pennant\Feature;
|
||||
|
||||
beforeEach(function () {
|
||||
config([
|
||||
'subscriptions.enabled' => true,
|
||||
'subscriptions.price_experiment.started_at' => '2026-06-01',
|
||||
'subscriptions.price_experiment.force_variant' => null,
|
||||
'subscriptions.price_experiment.variants.high' => [
|
||||
'monthly' => ['price' => 8.99, 'original_price' => null, 'lookup' => 'whisper_pro_monthly_v9'],
|
||||
'yearly' => ['price' => 53.94, 'original_price' => 107.88, 'lookup' => 'whisper_pro_yearly_v9'],
|
||||
],
|
||||
'subscriptions.plans.monthly.price' => 3.99,
|
||||
'subscriptions.plans.monthly.stripe_lookup_key' => 'whisper_pro_monthly',
|
||||
'subscriptions.plans.yearly.price' => 23.88,
|
||||
'subscriptions.plans.yearly.stripe_lookup_key' => 'whisper_pro_yearly',
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps users who registered before the experiment on the control price', function () {
|
||||
$user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-05-20')]);
|
||||
|
||||
expect(app(ExperimentOffer::class)->priceVariantFor($user))->toBe(PriceExperiment::LEGACY);
|
||||
});
|
||||
|
||||
it('treats a null (guest) scope as legacy', function () {
|
||||
expect((new PriceExperiment)->resolve(null))->toBe(PriceExperiment::LEGACY);
|
||||
});
|
||||
|
||||
it('treats everyone as legacy while the experiment is off', function () {
|
||||
config(['subscriptions.price_experiment.started_at' => null]);
|
||||
|
||||
$user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]);
|
||||
|
||||
expect(app(ExperimentOffer::class)->priceVariantFor($user))->toBe(PriceExperiment::LEGACY);
|
||||
});
|
||||
|
||||
it('pins every user to the forced winner price', function () {
|
||||
config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]);
|
||||
$offer = app(ExperimentOffer::class);
|
||||
|
||||
$legacy = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-05-01')]);
|
||||
$fresh = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]);
|
||||
|
||||
expect($offer->priceVariantFor($legacy))->toBe(PriceExperiment::HIGH)
|
||||
->and($offer->priceVariantFor($fresh))->toBe(PriceExperiment::HIGH);
|
||||
});
|
||||
|
||||
it('ignores an invalid forced variant', function () {
|
||||
config(['subscriptions.price_experiment.force_variant' => 'bogus']);
|
||||
|
||||
$user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-05-01')]);
|
||||
|
||||
expect(app(ExperimentOffer::class)->priceVariantFor($user))->toBe(PriceExperiment::LEGACY);
|
||||
});
|
||||
|
||||
it('splits post-start users across control and high and stays stable per user', function () {
|
||||
$offer = app(ExperimentOffer::class);
|
||||
$variants = [];
|
||||
|
||||
for ($i = 0; $i < 40; $i++) {
|
||||
$user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]);
|
||||
$assigned = $offer->priceVariantFor($user);
|
||||
$variants[] = $assigned;
|
||||
|
||||
Feature::flushCache();
|
||||
expect($offer->priceVariantFor($user))->toBe($assigned);
|
||||
}
|
||||
|
||||
expect(array_unique($variants))->toEqualCanonicalizing([
|
||||
PriceExperiment::CONTROL,
|
||||
PriceExperiment::HIGH,
|
||||
]);
|
||||
});
|
||||
|
||||
it('assigns the price independently of the trial experiment (salted hash)', function () {
|
||||
// For users the trial experiment buckets into pay_now, the price experiment
|
||||
// must still produce BOTH price arms — otherwise the two are collinear.
|
||||
$priceArmsWithinOneTrialArm = [];
|
||||
|
||||
for ($i = 0; $i < 200 && count(array_unique($priceArmsWithinOneTrialArm)) < 2; $i++) {
|
||||
$id = (string) Str::uuid();
|
||||
|
||||
if (SubscriptionExperiment::bucket($id) === SubscriptionExperiment::PAY_NOW) {
|
||||
$priceArmsWithinOneTrialArm[] = PriceExperiment::bucket($id);
|
||||
}
|
||||
}
|
||||
|
||||
expect(array_unique($priceArmsWithinOneTrialArm))->toEqualCanonicalizing([
|
||||
PriceExperiment::CONTROL,
|
||||
PriceExperiment::HIGH,
|
||||
]);
|
||||
});
|
||||
|
||||
it('applies the variant price and lookup key for a high-price user', function () {
|
||||
$offer = app(ExperimentOffer::class);
|
||||
$user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]);
|
||||
Feature::for($user)->activate(PriceExperiment::class, PriceExperiment::HIGH);
|
||||
|
||||
$plans = $offer->pricingPlansFor($user);
|
||||
|
||||
expect($plans['monthly']['price'])->toBe(8.99)
|
||||
->and($plans['monthly']['stripe_lookup_key'])->toBe('whisper_pro_monthly_v9')
|
||||
->and($plans['yearly']['price'])->toBe(53.94)
|
||||
->and($plans['yearly']['original_price'])->toBe(107.88)
|
||||
->and($offer->lookupKeyFor($user, 'monthly'))->toBe('whisper_pro_monthly_v9')
|
||||
->and($offer->lookupKeyFor($user, 'yearly'))->toBe('whisper_pro_yearly_v9');
|
||||
});
|
||||
|
||||
it('leaves control-price users on the config prices', function () {
|
||||
$offer = app(ExperimentOffer::class);
|
||||
$user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]);
|
||||
Feature::for($user)->activate(PriceExperiment::class, PriceExperiment::CONTROL);
|
||||
|
||||
$plans = $offer->pricingPlansFor($user);
|
||||
|
||||
expect($plans['monthly']['price'])->toBe(3.99)
|
||||
->and($plans['monthly']['stripe_lookup_key'])->toBe('whisper_pro_monthly')
|
||||
->and($offer->lookupKeyFor($user, 'monthly'))->toBe('whisper_pro_monthly');
|
||||
});
|
||||
|
||||
it('shows the assigned variant price on the paywall', function () {
|
||||
$user = User::factory()->onboarded()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]);
|
||||
Feature::for($user)->activate(PriceExperiment::class, PriceExperiment::HIGH);
|
||||
|
||||
$this->actingAs($user)
|
||||
->get(route('subscribe'))
|
||||
->assertOk()
|
||||
->assertInertia(fn ($page) => $page
|
||||
->component('subscription/paywall')
|
||||
->where('pricing.plans.monthly.price', 8.99)
|
||||
->where('pricing.plans.yearly.price', 53.94));
|
||||
});
|
||||
|
||||
it('charges the assigned variant price id at checkout, not a client-supplied one', function () {
|
||||
config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]);
|
||||
Cache::put('stripe_price_id:whisper_pro_monthly_v9', 'price_high_monthly', now()->addHour());
|
||||
|
||||
$checkout = Mockery::mock(Checkout::class);
|
||||
$checkout->shouldReceive('toResponse')->andReturn(new RedirectResponse('https://stripe.test/session'));
|
||||
|
||||
$builder = Mockery::mock(SubscriptionBuilder::class)->shouldIgnoreMissing();
|
||||
$builder->shouldReceive('checkout')->once()->andReturn($checkout);
|
||||
|
||||
$user = Mockery::mock(User::class)->shouldIgnoreMissing();
|
||||
$user->shouldReceive('hasVerifiedEmail')->andReturn(true);
|
||||
$user->shouldReceive('hasProPlan')->andReturn(false);
|
||||
$user->shouldReceive('newSubscription')
|
||||
->once()
|
||||
->with('default', 'price_high_monthly')
|
||||
->andReturn($builder);
|
||||
|
||||
$this->withoutMiddleware(HandleInertiaRequests::class);
|
||||
$this->actingAs($user);
|
||||
|
||||
$this->get(route('subscribe.checkout', ['plan' => 'monthly']))->assertRedirect();
|
||||
});
|
||||
|
|
@ -0,0 +1,262 @@
|
|||
<?php
|
||||
|
||||
use App\Features\PriceExperiment;
|
||||
use App\Models\AiConsent;
|
||||
use App\Models\BankingConnection;
|
||||
use App\Models\User;
|
||||
use App\Services\Stats\PriceExperimentFunnelCollector;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Artisan;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
use function Pest\Laravel\artisan;
|
||||
|
||||
beforeEach(function () {
|
||||
config([
|
||||
'subscriptions.enabled' => true,
|
||||
'subscriptions.price_experiment.started_at' => '2026-06-01',
|
||||
'subscriptions.price_experiment.force_variant' => null,
|
||||
'subscriptions.plans.monthly.trial_days' => 15,
|
||||
'ai_suggestions.report.excluded_emails' => [],
|
||||
]);
|
||||
Carbon::setTestNow(CarbonImmutable::parse('2026-06-30 12:00:00'));
|
||||
|
||||
// Seed the price→monthly-equivalent map so revenue is computed without Stripe:
|
||||
// control €3.99, high €8.99.
|
||||
Cache::put('experiment_funnel_monthly_equiv', ['price_control' => 399, 'price_high' => 899], now()->addHour());
|
||||
});
|
||||
|
||||
/**
|
||||
* Create a user whose id buckets into the wanted price variant, with an optional
|
||||
* default subscription, live (active) bank connections that drive the monthly cost,
|
||||
* extra soft-deleted connections that count as "ever activated" but cost nothing,
|
||||
* and optional AI consent.
|
||||
*
|
||||
* @param array{status: string, at: CarbonImmutable, price?: string, endsAt?: CarbonImmutable, trialEndsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription
|
||||
*/
|
||||
function priceUser(string $variant, CarbonImmutable $signup, ?array $subscription = null, int $connections = 0, int $deletedConnections = 0, bool $aiConsent = false): User
|
||||
{
|
||||
do {
|
||||
$id = (string) Str::uuid();
|
||||
} while (PriceExperiment::bucket($id) !== $variant);
|
||||
|
||||
$user = User::factory()->create(['id' => $id, 'created_at' => $signup]);
|
||||
|
||||
if ($subscription !== null) {
|
||||
$user->subscriptions()->create([
|
||||
'type' => 'default',
|
||||
'stripe_id' => 'sub_'.Str::random(12),
|
||||
'stripe_status' => $subscription['status'],
|
||||
'stripe_price' => $subscription['price'] ?? 'price_control',
|
||||
'created_at' => $subscription['at'],
|
||||
'ends_at' => $subscription['endsAt'] ?? null,
|
||||
'trial_ends_at' => $subscription['trialEndsAt'] ?? null,
|
||||
'refunded_at' => $subscription['refundedAt'] ?? null,
|
||||
]);
|
||||
}
|
||||
|
||||
if ($connections > 0) {
|
||||
BankingConnection::factory()->count($connections)->for($user)->create();
|
||||
}
|
||||
|
||||
if ($deletedConnections > 0) {
|
||||
BankingConnection::factory()->count($deletedConnections)->for($user)->create()
|
||||
->each(fn (BankingConnection $connection) => $connection->delete());
|
||||
}
|
||||
|
||||
if ($aiConsent) {
|
||||
AiConsent::factory()->for($user)->create();
|
||||
}
|
||||
|
||||
return $user;
|
||||
}
|
||||
|
||||
it('returns empty variants when the experiment has not started', function () {
|
||||
config(['subscriptions.price_experiment.started_at' => null]);
|
||||
|
||||
$report = app(PriceExperimentFunnelCollector::class)->collect();
|
||||
|
||||
expect($report['startedAt'])->toBeNull()
|
||||
->and($report['variants'][PriceExperiment::CONTROL]['assigned'])->toBe(0)
|
||||
->and($report['variants'][PriceExperiment::HIGH]['assigned'])->toBe(0);
|
||||
});
|
||||
|
||||
it('attributes users to the right price variant and counts the funnel', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05'); // mature (15d + 3d) by the test clock
|
||||
|
||||
priceUser(PriceExperiment::CONTROL, $signup);
|
||||
priceUser(PriceExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay(), 'price' => 'price_control']);
|
||||
priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup->addDay(), 'price' => 'price_high']);
|
||||
|
||||
$variants = app(PriceExperimentFunnelCollector::class)->collect()['variants'];
|
||||
|
||||
expect($variants[PriceExperiment::CONTROL]['assigned'])->toBe(2)
|
||||
->and($variants[PriceExperiment::CONTROL]['subscribed'])->toBe(1)
|
||||
->and($variants[PriceExperiment::CONTROL]['activeMature'])->toBe(1)
|
||||
->and($variants[PriceExperiment::CONTROL]['convertedMature'])->toBe(1)
|
||||
->and($variants[PriceExperiment::CONTROL]['conversionRate'])->toBe(0.5)
|
||||
->and($variants[PriceExperiment::HIGH]['assigned'])->toBe(1)
|
||||
->and($variants[PriceExperiment::HIGH]['activeMature'])->toBe(1);
|
||||
});
|
||||
|
||||
it('books MRR at each variant price and reports contribution margin per assigned user', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// High arm: one payer at €8.99 with 1 active connection (cost €0.40).
|
||||
priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_high'], connections: 1);
|
||||
// High arm: one assigned non-payer, no connections.
|
||||
priceUser(PriceExperiment::HIGH, $signup);
|
||||
|
||||
$high = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::HIGH];
|
||||
|
||||
expect($high['mrrCents'])->toBe(899) // €8.99 at the high price
|
||||
->and($high['costCents'])->toBe(40) // 1 active connection
|
||||
->and($high['contributionMarginCents'])->toBe(859) // 899 − 40
|
||||
->and($high['assignedMature'])->toBe(2)
|
||||
->and($high['cmPerAssignedCents'])->toBe(430) // round(859 / 2)
|
||||
->and($high['connectionsPerPayer'])->toBe(1.0);
|
||||
});
|
||||
|
||||
it('counts revoked connections as activation but charges cost only for live ones', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// 1 live connection + 2 soft-deleted: activated (ever connected), but the
|
||||
// monthly cost is only the 1 live connection — the stock-vs-flow fix.
|
||||
priceUser(PriceExperiment::CONTROL, $signup, connections: 1, deletedConnections: 2);
|
||||
|
||||
$control = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::CONTROL];
|
||||
|
||||
expect($control['activated'])->toBe(1)
|
||||
->and($control['activatedMature'])->toBe(1)
|
||||
->and($control['costCents'])->toBe(40); // 1 live × 40, not 3 × 40
|
||||
});
|
||||
|
||||
it('leaves young cohorts out of the matured metrics', function () {
|
||||
priceUser(PriceExperiment::HIGH, CarbonImmutable::now()->subDays(5), [
|
||||
'status' => 'active', 'at' => CarbonImmutable::now()->subDays(5), 'price' => 'price_high',
|
||||
]);
|
||||
|
||||
$high = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::HIGH];
|
||||
|
||||
expect($high['assigned'])->toBe(1)
|
||||
->and($high['assignedMature'])->toBe(0) // 5d < 18d window
|
||||
->and($high['conversionRate'])->toBeNull();
|
||||
});
|
||||
|
||||
it('keeps the split even when a winner is forced, instead of collapsing onto one variant', function () {
|
||||
config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]);
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
priceUser(PriceExperiment::CONTROL, $signup);
|
||||
priceUser(PriceExperiment::HIGH, $signup);
|
||||
|
||||
$variants = app(PriceExperimentFunnelCollector::class)->collect()['variants'];
|
||||
|
||||
expect($variants[PriceExperiment::CONTROL]['assigned'])->toBe(1)
|
||||
->and($variants[PriceExperiment::HIGH]['assigned'])->toBe(1);
|
||||
});
|
||||
|
||||
it('prints the primary CM/user (Welch) and conversion guardrail blocks but withholds the verdict at small n', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
for ($i = 0; $i < 3; $i++) {
|
||||
priceUser(PriceExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_control']);
|
||||
priceUser(PriceExperiment::CONTROL, $signup);
|
||||
priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_high']);
|
||||
priceUser(PriceExperiment::HIGH, $signup);
|
||||
}
|
||||
|
||||
Artisan::call('stats:price-experiment-funnel', ['--no-discord' => true]);
|
||||
$output = Artisan::output();
|
||||
|
||||
expect($output)->toContain('PRIMARY')
|
||||
->and($output)->toContain('GUARDRAIL')
|
||||
->and($output)->toContain('Fisher exact')
|
||||
->and($output)->toContain('verdict withheld') // 6 per arm < MIN_WELCH_N
|
||||
->and($output)->not->toContain('α=0.05 ->'); // no significance verdict at small n
|
||||
});
|
||||
|
||||
it('emits a Welch significance verdict once both arms clear the small-sample floor', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// 30 per arm (== MIN_WELCH_N), half net-active payers, so there is real variance.
|
||||
for ($i = 0; $i < 30; $i++) {
|
||||
$sub = $i % 2 === 0 ? ['status' => 'active', 'at' => $signup] : null;
|
||||
priceUser(PriceExperiment::CONTROL, $signup, $sub === null ? null : [...$sub, 'price' => 'price_control']);
|
||||
priceUser(PriceExperiment::HIGH, $signup, $sub === null ? null : [...$sub, 'price' => 'price_high']);
|
||||
}
|
||||
|
||||
Artisan::call('stats:price-experiment-funnel', ['--no-discord' => true]);
|
||||
$output = Artisan::output();
|
||||
|
||||
expect($output)->toContain('α=0.05 ->')
|
||||
->and($output)->not->toContain('verdict withheld');
|
||||
});
|
||||
|
||||
it('surfaces an unmapped-price warning in the report, not just the logs', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// Net-active sub on a price id the seeded map does not know → MRR silently 0.
|
||||
priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_rotated_old']);
|
||||
|
||||
Artisan::call('stats:price-experiment-funnel', ['--no-discord' => true]);
|
||||
$output = Artisan::output();
|
||||
|
||||
expect($output)->toContain('UNMAPPED PRICES')
|
||||
->and($output)->toContain('price_rotated_old');
|
||||
});
|
||||
|
||||
it('flags a sample-ratio mismatch when the assignment split is lopsided', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
for ($i = 0; $i < 10; $i++) {
|
||||
priceUser(PriceExperiment::CONTROL, $signup);
|
||||
}
|
||||
priceUser(PriceExperiment::HIGH, $signup);
|
||||
|
||||
artisan('stats:price-experiment-funnel', ['--no-discord' => true])
|
||||
->expectsOutputToContain('SRM')
|
||||
->expectsOutputToContain('IMBALANCE')
|
||||
->assertSuccessful();
|
||||
});
|
||||
|
||||
it('does not flag SRM when the split is balanced', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
for ($i = 0; $i < 4; $i++) {
|
||||
priceUser(PriceExperiment::CONTROL, $signup);
|
||||
priceUser(PriceExperiment::HIGH, $signup);
|
||||
}
|
||||
|
||||
artisan('stats:price-experiment-funnel', ['--no-discord' => true])
|
||||
->expectsOutputToContain('SRM')
|
||||
->doesntExpectOutputToContain('IMBALANCE')
|
||||
->assertSuccessful();
|
||||
});
|
||||
|
||||
it('posts the price experiment embed to discord with the monitoring-only warning', function () {
|
||||
config(['services.discord.ai_cohort_webhook_url' => 'https://discord.test/hook']);
|
||||
Http::fake(['discord.test/*' => Http::response('', 204)]);
|
||||
|
||||
priceUser(PriceExperiment::CONTROL, CarbonImmutable::parse('2026-06-05'), [
|
||||
'status' => 'active', 'at' => CarbonImmutable::parse('2026-06-05'), 'price' => 'price_control',
|
||||
]);
|
||||
|
||||
artisan('stats:price-experiment-funnel')->assertSuccessful();
|
||||
|
||||
Http::assertSent(fn ($request) => $request->url() === 'https://discord.test/hook'
|
||||
&& str_contains($request['embeds'][0]['title'], 'Price Experiment')
|
||||
&& str_contains(json_encode($request['embeds'][0]), 'MONITORING ONLY'));
|
||||
});
|
||||
|
||||
it('does not post when the experiment has not started', function () {
|
||||
config(['subscriptions.price_experiment.started_at' => null]);
|
||||
Http::fake();
|
||||
|
||||
artisan('stats:price-experiment-funnel')->assertSuccessful();
|
||||
|
||||
Http::assertNothingSent();
|
||||
});
|
||||
|
|
@ -68,6 +68,8 @@ beforeEach(function () {
|
|||
],
|
||||
'subscriptions.products.pro' => 'prod_test123',
|
||||
'cashier.currency' => 'eur',
|
||||
// Base-plan tests isolate the plans loop; variant syncing has its own test.
|
||||
'subscriptions.price_experiment.variants' => [],
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
@ -149,3 +151,25 @@ test('returns success and warns when no plans are configured', function () {
|
|||
->expectsOutputToContain('No plans found')
|
||||
->assertSuccessful();
|
||||
});
|
||||
|
||||
test('syncs the price-experiment variant tiers alongside the base plans', function () {
|
||||
config([
|
||||
'subscriptions.price_experiment.variants' => [
|
||||
'high' => [
|
||||
'monthly' => ['price' => 8.99, 'lookup' => 'whisper_pro_monthly_v9'],
|
||||
'yearly' => ['price' => 53.94, 'lookup' => 'whisper_pro_yearly_v9'],
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
// Base plans already in sync; both high tiers are new → 2 created, 2 skipped.
|
||||
bindMockStripeClientForSync([
|
||||
'whisper_pro_monthly' => makeStripePrice('price_monthly', 399, 'eur', 'month'),
|
||||
'whisper_pro_yearly' => makeStripePrice('price_yearly', 2388, 'eur', 'year'),
|
||||
]);
|
||||
|
||||
$this->artisan('stripe:sync-prices')
|
||||
->expectsOutputToContain('(price: high)')
|
||||
->expectsOutputToContain('2 created, 0 updated, 2 skipped')
|
||||
->assertSuccessful();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
<?php
|
||||
|
||||
use App\Services\Stats\SampleRatioMismatch;
|
||||
|
||||
it('reports p=1 for an exactly even split', function () {
|
||||
$result = (new SampleRatioMismatch)->evenSplit(50, 50);
|
||||
|
||||
expect($result['chiSq'])->toBe(0.0)
|
||||
->and($result['p'])->toEqualWithDelta(1.0, 1e-6);
|
||||
});
|
||||
|
||||
it('returns a neutral result for empty arms', function () {
|
||||
$result = (new SampleRatioMismatch)->evenSplit(0, 0);
|
||||
|
||||
expect($result['chiSq'])->toBe(0.0)
|
||||
->and($result['p'])->toBe(1.0);
|
||||
});
|
||||
|
||||
it('matches the known chi-square for a 60/40 split of 100', function () {
|
||||
// χ² = ((60−50)² + (40−50)²)/50 = 4 → p = 2·(1 − Φ(2)) = 0.0455.
|
||||
$result = (new SampleRatioMismatch)->evenSplit(60, 40);
|
||||
|
||||
expect($result['chiSq'])->toEqualWithDelta(4.0, 1e-9)
|
||||
->and($result['p'])->toEqualWithDelta(0.0455, 1e-3);
|
||||
});
|
||||
|
||||
it('flags a badly lopsided split with a tiny p', function () {
|
||||
$result = (new SampleRatioMismatch)->evenSplit(90, 10);
|
||||
|
||||
expect($result['chiSq'])->toEqualWithDelta(64.0, 1e-9)
|
||||
->and($result['p'])->toBeLessThan(0.001);
|
||||
});
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
<?php
|
||||
|
||||
use App\Services\Stats\WelchTTest;
|
||||
|
||||
it('returns no difference and p=1 for identical arms', function () {
|
||||
$result = (new WelchTTest)->compare(10.0, 4.0, 100, 10.0, 4.0, 100);
|
||||
|
||||
expect($result['diff'])->toBe(0.0)
|
||||
->and($result['t'])->toBe(0.0)
|
||||
->and($result['p'])->toEqualWithDelta(1.0, 1e-6); // erf approximation, |error| < 1.5e-7
|
||||
});
|
||||
|
||||
it('yields a two-sided p of ~0.317 when the difference equals one standard error', function () {
|
||||
// varA/nA = varB/nB = 0.5 → seSq = 1, se = 1; diff = 1 → t = 1.
|
||||
// Two-sided normal p = 2·(1 − Φ(1)) = 0.3173, validating the erf/normalCdf path.
|
||||
$result = (new WelchTTest)->compare(1.0, 50.0, 100, 0.0, 50.0, 100);
|
||||
|
||||
expect($result['t'])->toEqualWithDelta(1.0, 1e-9)
|
||||
->and($result['se'])->toEqualWithDelta(1.0, 1e-9)
|
||||
->and($result['p'])->toEqualWithDelta(0.3173, 1e-3);
|
||||
});
|
||||
|
||||
it('finds a large separation highly significant', function () {
|
||||
$result = (new WelchTTest)->compare(10.0, 4.0, 100, 8.0, 4.0, 100);
|
||||
|
||||
expect($result['t'])->toEqualWithDelta(7.071, 1e-2)
|
||||
->and($result['p'])->toBeLessThan(0.001);
|
||||
});
|
||||
|
||||
it('defers (p=1) when an arm has too few observations to estimate variance', function () {
|
||||
$result = (new WelchTTest)->compare(5.0, 0.0, 1, 0.0, 0.0, 1);
|
||||
|
||||
expect($result['p'])->toBe(1.0);
|
||||
});
|
||||
Loading…
Reference in New Issue