fix(stats): attribute the funnel by deterministic bucket, not resolved flag
Reading Feature::value() routed attribution through the before() hook, so setting SUBSCRIPTION_EXPERIMENT_FORCE_VARIANT (the documented winner-rollout switch) collapsed the whole weekly report onto one variant. Attribute by SubscriptionExperiment::bucket() instead — the deterministic assignment every in-window user was served — which is immune to force_variant and to started_at moves, stops the report writing Pennant rows as a side effect, and makes the feature's own docblock true again.
This commit is contained in:
parent
2eab6f2a76
commit
af03a179af
|
|
@ -9,7 +9,6 @@ use Illuminate\Support\Facades\Cache;
|
|||
use Illuminate\Support\Facades\Log;
|
||||
use Laravel\Cashier\Cashier;
|
||||
use Laravel\Cashier\Subscription;
|
||||
use Laravel\Pennant\Feature;
|
||||
|
||||
class ExperimentFunnelCollector
|
||||
{
|
||||
|
|
@ -21,12 +20,13 @@ class ExperimentFunnelCollector
|
|||
|
||||
/**
|
||||
* Per-variant funnel for the trial/pricing experiment. Users are attributed
|
||||
* by the variant Pennant resolved for them — the same value the runtime
|
||||
* served at checkout/paywall — so the report can't drift from what users
|
||||
* actually experienced (including any QA override or a legacy bucket that
|
||||
* predates the experiment). "Net active" is a live, non-refunded
|
||||
* subscription — an exact, heuristic-free metric that is comparable across
|
||||
* variants once each cohort clears its own decision window.
|
||||
* by SubscriptionExperiment::bucket() — the deterministic crc32 split that is
|
||||
* the single source of truth for assignment — over the in-window signups the
|
||||
* query selects, so it matches the variant each user was served without being
|
||||
* perturbed by the force_variant rollout hook (which pins every user to the
|
||||
* winner once decided) or by Pennant store drift. "Net active" is a live,
|
||||
* non-refunded subscription — an exact, heuristic-free metric that is
|
||||
* comparable across variants once each cohort clears its own decision window.
|
||||
*
|
||||
* The funnel is assigned → activated → carded (subscribed) → net-paying:
|
||||
* "activated" = the user connected a bank or enabled AI, i.e. triggered the
|
||||
|
|
@ -111,10 +111,15 @@ class ExperimentFunnelCollector
|
|||
'aiConsents as ai_consent_count',
|
||||
])
|
||||
->chunkById(500, function ($users) use (&$variants, &$missingPrices, $windows, $now, $monthlyEquiv, $costPerConnectionCents): void {
|
||||
Feature::for($users)->load([SubscriptionExperiment::class]);
|
||||
|
||||
foreach ($users as $user) {
|
||||
$variant = Feature::for($user)->value(SubscriptionExperiment::class);
|
||||
// Attribute by the deterministic bucket (the single source of
|
||||
// truth in SubscriptionExperiment), not the resolved Pennant
|
||||
// value: the latter is short-circuited by the force_variant
|
||||
// rollout hook, which would collapse every user onto one
|
||||
// variant once a winner is pinned. Every queried user is
|
||||
// in-window, so bucket() equals the variant they were served,
|
||||
// and reading it avoids writing Pennant rows as a side effect.
|
||||
$variant = SubscriptionExperiment::bucket((string) $user->id);
|
||||
|
||||
if (! isset($variants[$variant])) {
|
||||
continue;
|
||||
|
|
|
|||
|
|
@ -174,6 +174,23 @@ it('computes connection cost, wasted burn and contribution margin', function ()
|
|||
->and($control['activationToPaidRate'])->toBe(0.5); // 1 paid ÷ 2 activated
|
||||
});
|
||||
|
||||
it('keeps the A/B/C split even when a winner is forced, instead of collapsing onto one variant', function () {
|
||||
// force_variant pins the runtime to one variant; the report must still show
|
||||
// the real historical split, not attribute everyone to the forced winner.
|
||||
config(['subscriptions.experiment.force_variant' => SubscriptionExperiment::PAY_NOW]);
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup);
|
||||
experimentUser(SubscriptionExperiment::REDUCED_TRIAL, $signup);
|
||||
experimentUser(SubscriptionExperiment::PAY_NOW, $signup);
|
||||
|
||||
$variants = app(ExperimentFunnelCollector::class)->collect()['variants'];
|
||||
|
||||
expect($variants[SubscriptionExperiment::CONTROL]['assigned'])->toBe(1)
|
||||
->and($variants[SubscriptionExperiment::REDUCED_TRIAL]['assigned'])->toBe(1)
|
||||
->and($variants[SubscriptionExperiment::PAY_NOW]['assigned'])->toBe(1);
|
||||
});
|
||||
|
||||
it('warns instead of silently zeroing MRR when a paid sub is on an unmapped (rotated) price', function () {
|
||||
Log::spy();
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
|
|
|||
Loading…
Reference in New Issue