diff --git a/app/Services/Stats/ExperimentFunnelCollector.php b/app/Services/Stats/ExperimentFunnelCollector.php index 4ff4ebdd..2a6bf9eb 100644 --- a/app/Services/Stats/ExperimentFunnelCollector.php +++ b/app/Services/Stats/ExperimentFunnelCollector.php @@ -9,7 +9,6 @@ use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Log; use Laravel\Cashier\Cashier; use Laravel\Cashier\Subscription; -use Laravel\Pennant\Feature; class ExperimentFunnelCollector { @@ -21,12 +20,13 @@ class ExperimentFunnelCollector /** * Per-variant funnel for the trial/pricing experiment. Users are attributed - * by the variant Pennant resolved for them — the same value the runtime - * served at checkout/paywall — so the report can't drift from what users - * actually experienced (including any QA override or a legacy bucket that - * predates the experiment). "Net active" is a live, non-refunded - * subscription — an exact, heuristic-free metric that is comparable across - * variants once each cohort clears its own decision window. + * by SubscriptionExperiment::bucket() — the deterministic crc32 split that is + * the single source of truth for assignment — over the in-window signups the + * query selects, so it matches the variant each user was served without being + * perturbed by the force_variant rollout hook (which pins every user to the + * winner once decided) or by Pennant store drift. "Net active" is a live, + * non-refunded subscription — an exact, heuristic-free metric that is + * comparable across variants once each cohort clears its own decision window. * * The funnel is assigned → activated → carded (subscribed) → net-paying: * "activated" = the user connected a bank or enabled AI, i.e. triggered the @@ -111,10 +111,15 @@ class ExperimentFunnelCollector 'aiConsents as ai_consent_count', ]) ->chunkById(500, function ($users) use (&$variants, &$missingPrices, $windows, $now, $monthlyEquiv, $costPerConnectionCents): void { - Feature::for($users)->load([SubscriptionExperiment::class]); - foreach ($users as $user) { - $variant = Feature::for($user)->value(SubscriptionExperiment::class); + // Attribute by the deterministic bucket (the single source of + // truth in SubscriptionExperiment), not the resolved Pennant + // value: the latter is short-circuited by the force_variant + // rollout hook, which would collapse every user onto one + // variant once a winner is pinned. Every queried user is + // in-window, so bucket() equals the variant they were served, + // and reading it avoids writing Pennant rows as a side effect. + $variant = SubscriptionExperiment::bucket((string) $user->id); if (! isset($variants[$variant])) { continue; diff --git a/tests/Feature/SendExperimentFunnelReportCommandTest.php b/tests/Feature/SendExperimentFunnelReportCommandTest.php index c4111e7d..79d63cb3 100644 --- a/tests/Feature/SendExperimentFunnelReportCommandTest.php +++ b/tests/Feature/SendExperimentFunnelReportCommandTest.php @@ -174,6 +174,23 @@ it('computes connection cost, wasted burn and contribution margin', function () ->and($control['activationToPaidRate'])->toBe(0.5); // 1 paid ÷ 2 activated }); +it('keeps the A/B/C split even when a winner is forced, instead of collapsing onto one variant', function () { + // force_variant pins the runtime to one variant; the report must still show + // the real historical split, not attribute everyone to the forced winner. + config(['subscriptions.experiment.force_variant' => SubscriptionExperiment::PAY_NOW]); + $signup = CarbonImmutable::parse('2026-06-05'); + + experimentUser(SubscriptionExperiment::CONTROL, $signup); + experimentUser(SubscriptionExperiment::REDUCED_TRIAL, $signup); + experimentUser(SubscriptionExperiment::PAY_NOW, $signup); + + $variants = app(ExperimentFunnelCollector::class)->collect()['variants']; + + expect($variants[SubscriptionExperiment::CONTROL]['assigned'])->toBe(1) + ->and($variants[SubscriptionExperiment::REDUCED_TRIAL]['assigned'])->toBe(1) + ->and($variants[SubscriptionExperiment::PAY_NOW]['assigned'])->toBe(1); +}); + it('warns instead of silently zeroing MRR when a paid sub is on an unmapped (rotated) price', function () { Log::spy(); $signup = CarbonImmutable::parse('2026-06-05');