diff --git a/app/Actions/Fortify/CreateNewUser.php b/app/Actions/Fortify/CreateNewUser.php index 6a7e2f76..b8b0eafd 100644 --- a/app/Actions/Fortify/CreateNewUser.php +++ b/app/Actions/Fortify/CreateNewUser.php @@ -4,6 +4,7 @@ namespace App\Actions\Fortify; use App\Enums\Locale; use App\Models\User; +use App\Services\Subscriptions\PriceExperiment; use Illuminate\Support\Facades\Validator; use Illuminate\Validation\Rule; use Laravel\Fortify\Contracts\CreatesNewUsers; @@ -40,6 +41,10 @@ class CreateNewUser implements CreatesNewUsers 'password' => $input['password'], 'locale' => Locale::detectFromHeader(request()->header('Accept-Language'))->value, 'timezone' => $this->normalizeTimezone($input['timezone'] ?? null), + // Freeze the arm this visitor was quoted as an anonymous browser, so + // the price on the landing is the price at checkout. Null when they + // arrived without a cookie: those users pay the control price. + 'price_arm' => PriceExperiment::sanitize(request()->cookie(PriceExperiment::COOKIE)), ]); if (! config('mail.email_verification_enabled')) { diff --git a/app/Http/Middleware/AssignPriceExperimentArm.php b/app/Http/Middleware/AssignPriceExperimentArm.php new file mode 100644 index 00000000..696cd194 --- /dev/null +++ b/app/Http/Middleware/AssignPriceExperimentArm.php @@ -0,0 +1,46 @@ +shouldDraw($request)) { + $arm = PriceExperiment::draw(); + + $request->cookies->set(PriceExperiment::COOKIE, $arm); + Cookie::queue(PriceExperiment::COOKIE, $arm, minutes: 60 * 24 * 365); + } + + return $next($request); + } + + /** + * Only anonymous visitors browsing a page are drawn: a signed-in user already + * carries their arm on the user row. This middleware is in the web group, so + * API traffic never reaches it. + */ + private function shouldDraw(Request $request): bool + { + return PriceExperiment::isRunning() + && $request->user() === null + && $request->isMethod('GET') + && PriceExperiment::sanitize($request->cookie(PriceExperiment::COOKIE)) === null; + } +} diff --git a/app/Http/Middleware/HandleInertiaRequests.php b/app/Http/Middleware/HandleInertiaRequests.php index 2893eef2..4c4779ca 100644 --- a/app/Http/Middleware/HandleInertiaRequests.php +++ b/app/Http/Middleware/HandleInertiaRequests.php @@ -93,7 +93,7 @@ class HandleInertiaRequests extends Middleware 'subscriptionsEnabled' => config('subscriptions.enabled', false), 'aiCategorizationUpsellRate' => (int) config('ai_categorization.upsell_sample_rate'), 'pricing' => [ - 'plans' => PriceExperiment::plansFor($user), + 'plans' => PriceExperiment::plansFor($user, $request->cookie(PriceExperiment::COOKIE)), 'defaultPlan' => config('subscriptions.default_plan', 'monthly'), 'bestValuePlan' => config('subscriptions.best_value_plan', null), 'promo' => config('subscriptions.promo', []), diff --git a/app/Models/User.php b/app/Models/User.php index ab677561..7f2a8f72 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -33,6 +33,7 @@ use Laravel\Sanctum\HasApiTokens; * @property ?Carbon $last_active_at * @property ?Carbon $transactions_last_visited_at * @property ?Carbon $ai_consent_prompt_dismissed_at + * @property ?string $price_arm */ class User extends Authenticatable implements HasLocalePreference, MustVerifyEmail { @@ -55,6 +56,7 @@ class User extends Authenticatable implements HasLocalePreference, MustVerifyEma 'locale', 'timezone', 'current_space_id', + 'price_arm', ]; /** @@ -72,6 +74,7 @@ class User extends Authenticatable implements HasLocalePreference, MustVerifyEma 'pm_last_four', 'trial_ends_at', 'encryption_salt', + 'price_arm', ]; /** diff --git a/app/Services/Subscriptions/PriceExperiment.php b/app/Services/Subscriptions/PriceExperiment.php index 06f44736..cc96fcf4 100644 --- a/app/Services/Subscriptions/PriceExperiment.php +++ b/app/Services/Subscriptions/PriceExperiment.php @@ -3,27 +3,30 @@ namespace App\Services\Subscriptions; use App\Models\User; -use Carbon\CarbonImmutable; /** * A/B split on the price of the paid plan: `control` keeps the plans.* prices, - * `high` swaps in the variant tier. Users who registered before `started_at` — - * and everyone while it is null — are `legacy` and pay the control price, so - * this is inert until the experiment is switched on. + * `high` swaps in the variant tier. * - * ponytail: the assignment is a pure salted hash of the user id, not a stored - * Pennant feature. Nothing has to be persisted, read back or purged afterwards, - * and a report can reproduce the split in SQL with CRC32(CONCAT('price:', id)). - * Move it to Pennant only if an experiment ever needs a non-deterministic or - * hand-overridden per-user assignment. + * The arm is drawn for an **anonymous visitor** on their first page view and kept + * in a cookie, then copied onto the user row at registration. It has to work that + * way round: the landing quotes a price before anyone has an account, so assigning + * at registration would advertise the control price to everyone and then switch + * half of them to the high one — measuring the annoyance of a price that moved + * rather than the price itself, and hiding the visitors who would never have + * signed up at the higher price at all. * - * @api The variant names are the vocabulary the experiment is configured and read - * with — they are the accepted values of PRICE_EXPERIMENT_FORCE_VARIANT and - * what a funnel report attributes users by — even though production only - * calls plansFor()/lookupKeyFor(). + * Anyone without an arm — registered before the experiment, cookies blocked, + * arrived straight at a deep link — is `legacy` and pays the control price. + * + * @api The arm names are the vocabulary the experiment is configured and read + * with: the accepted values of PRICE_EXPERIMENT_FORCE_VARIANT, the contents + * of users.price_arm, and what a funnel report groups by. */ class PriceExperiment { + public const COOKIE = 'price_arm'; + public const LEGACY = 'legacy'; public const CONTROL = 'control'; @@ -31,50 +34,60 @@ class PriceExperiment public const HIGH = 'high'; /** - * The 'price:' salt decouples this split from any other crc32-based split on - * the same user id, so experiments never share buckets. + * Whether new visitors should still be drawn into the split. A forced variant + * means a winner is being rolled out to everyone, so the split is over even + * though the start date is still set. */ - public static function variantFor(User $user): string + public static function isRunning(): bool { - $forced = config('subscriptions.price_experiment.force_variant'); - - if (in_array($forced, [self::CONTROL, self::HIGH], true)) { - return $forced; - } - - $startedAt = config('subscriptions.price_experiment.started_at'); - // blank(), not === null: an empty PRICE_EXPERIMENT_STARTED_AT in the env // reads back as '', and treating that as a start date would launch the - // experiment — charging the high price — on an ops typo. Same for a user - // with no signup date: fall back to the price they already know. - if (blank($startedAt) || $user->created_at === null) { - return self::LEGACY; - } - - if ($user->created_at->lt(CarbonImmutable::parse($startedAt))) { - return self::LEGACY; - } - - return crc32('price:'.$user->getKey()) % 2 === 0 ? self::CONTROL : self::HIGH; + // experiment — charging the high price — on an ops typo. + return filled(config('subscriptions.price_experiment.started_at')) + && self::forcedArm() === null; } /** - * The plans config with the user's variant applied: price, original_price and + * A fresh 50/50 draw for a visitor we have not seen before. Random rather than + * a hash: there is no stable identifier to hash before the user exists. + */ + public static function draw(): string + { + return random_int(0, 1) === 0 ? self::CONTROL : self::HIGH; + } + + /** + * A stored or cookie value narrowed to a real arm, or null. Guards the column + * and the cookie alike, both of which a user can put anything into. + */ + public static function sanitize(?string $arm): ?string + { + return in_array($arm, [self::CONTROL, self::HIGH], true) ? $arm : null; + } + + /** + * The arm to price a request with: a signed-in user keeps the arm stored at + * registration, a guest gets the one drawn into their cookie. + */ + public static function armFor(?User $user, ?string $cookieArm = null): string + { + return self::forcedArm() + ?? self::sanitize($user !== null ? $user->price_arm : $cookieArm) + ?? self::LEGACY; + } + + /** + * The plans config with the request's arm applied: price, original_price and * Stripe lookup key. Feeds both the shared pricing prop and checkout, so what - * is shown is always what is charged. Guests get the control config. + * is shown is always what is charged. * * @return array> */ - public static function plansFor(?User $user): array + public static function plansFor(?User $user, ?string $cookieArm = null): array { $plans = (array) config('subscriptions.plans', []); - - if ($user === null) { - return $plans; - } - - $overrides = (array) config('subscriptions.price_experiment.variants.'.self::variantFor($user), []); + $arm = self::armFor($user, $cookieArm); + $overrides = (array) config("subscriptions.price_experiment.variants.{$arm}", []); foreach ($overrides as $planKey => $override) { if (! isset($plans[$planKey])) { @@ -90,11 +103,21 @@ class PriceExperiment } /** - * Stripe lookup key to charge for a plan, resolved from the user's variant - * server-side and never from the request, so nobody can pick the cheap price. + * Stripe lookup key to charge for a plan, resolved from the user's stored arm + * server-side and never from the request, so nobody can pick the cheap price + * by editing their cookie after signing up. */ public static function lookupKeyFor(User $user, string $planKey): string { return (string) (self::plansFor($user)[$planKey]['stripe_lookup_key'] ?? ''); } + + /** + * The winner pinned via PRICE_EXPERIMENT_FORCE_VARIANT, applied to everyone — + * visitors and existing users — so a rollout needs no deploy and no backfill. + */ + private static function forcedArm(): ?string + { + return self::sanitize(config('subscriptions.price_experiment.force_variant')); + } } diff --git a/bootstrap/app.php b/bootstrap/app.php index c8945910..2c8dae85 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -1,5 +1,6 @@ web(append: [ HandleAppearance::class, SetLocale::class, + // Before HandleInertiaRequests: it draws the visitor's price arm onto + // the request, which the shared pricing prop then reads. + AssignPriceExperimentArm::class, SetSentryUser::class, HandleInertiaRequests::class, AddLinkHeadersForPreloadedAssets::class, diff --git a/config/subscriptions.php b/config/subscriptions.php index 8318ee6e..32d32907 100644 --- a/config/subscriptions.php +++ b/config/subscriptions.php @@ -20,11 +20,14 @@ return [ | Price Experiment |-------------------------------------------------------------------------- | - | A/B test on the price of the paid plan. Users who register on or after - | `started_at` are split 50/50 into `control` (the plans.* prices below) and - | `high` (the variant tier); anyone older keeps the control price. While - | `started_at` is null the experiment is off. Set `force_variant` to - | control/high to roll a winner out to everyone without a deploy. + | A/B test on the price of the paid plan. Anonymous visitors are drawn 50/50 + | into `control` (the plans.* prices below) and `high` (the variant tier) on + | their first page view, and the arm is frozen onto users.price_arm when they + | register — so the landing quotes the price they will actually be charged. + | Anyone without an arm keeps the control price. While `started_at` is blank + | the experiment is off and nobody is drawn. Set `force_variant` to + | control/high to roll a winner out to everyone without a deploy; that also + | ends the split. | | Each variant needs its own Stripe price — run `php artisan stripe:sync-prices` | BEFORE setting `started_at`. The yearly price is the monthly × 6, matching diff --git a/database/migrations/2026_08_12_142520_add_price_arm_to_users_table.php b/database/migrations/2026_08_12_142520_add_price_arm_to_users_table.php new file mode 100644 index 00000000..e41a519d --- /dev/null +++ b/database/migrations/2026_08_12_142520_add_price_arm_to_users_table.php @@ -0,0 +1,28 @@ +string('price_arm')->nullable()->after('locale'); + }); + } + + public function down(): void + { + Schema::table('users', function (Blueprint $table) { + $table->dropColumn('price_arm'); + }); + } +}; diff --git a/tests/Feature/PriceExperimentTest.php b/tests/Feature/PriceExperimentTest.php index c76ecb4a..662e64d2 100644 --- a/tests/Feature/PriceExperimentTest.php +++ b/tests/Feature/PriceExperimentTest.php @@ -3,10 +3,8 @@ use App\Http\Middleware\HandleInertiaRequests; use App\Models\User; use App\Services\Subscriptions\PriceExperiment; -use Carbon\CarbonImmutable; use Illuminate\Http\RedirectResponse; use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Str; use Laravel\Cashier\Checkout; use Laravel\Cashier\SubscriptionBuilder; @@ -26,77 +24,59 @@ beforeEach(function () { ]); }); -it('keeps users who registered before the experiment on the control price', function () { - $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-05-20')]); +it('treats a user with no stored arm as legacy', function () { + $user = User::factory()->create(['price_arm' => null]); - expect(PriceExperiment::variantFor($user))->toBe(PriceExperiment::LEGACY); + expect(PriceExperiment::armFor($user))->toBe(PriceExperiment::LEGACY); }); -it('treats everyone as legacy while the experiment is off', function (?string $startedAt) { +it('ignores a stored arm that is not a real variant', function () { + $user = User::factory()->create(['price_arm' => 'bogus']); + + expect(PriceExperiment::armFor($user))->toBe(PriceExperiment::LEGACY); +}); + +it('is not running while the experiment is off', function (?string $startedAt) { // An empty PRICE_EXPERIMENT_STARTED_AT in the env reads back as '', so an // ops typo must not launch the experiment and start charging the high price. config(['subscriptions.price_experiment.started_at' => $startedAt]); - $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); - - expect(PriceExperiment::variantFor($user))->toBe(PriceExperiment::LEGACY); + expect(PriceExperiment::isRunning())->toBeFalse(); })->with([null, '', ' ']); -it('pins every user to the forced winner price', function () { +it('stops drawing new visitors once a winner is forced', function () { config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]); - $legacy = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-05-01')]); - $fresh = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); + expect(PriceExperiment::isRunning())->toBeFalse(); +}); - expect(PriceExperiment::variantFor($legacy))->toBe(PriceExperiment::HIGH) - ->and(PriceExperiment::variantFor($fresh))->toBe(PriceExperiment::HIGH); +it('pins everyone to the forced winner, visitors and users alike', function () { + config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]); + + $legacy = User::factory()->create(['price_arm' => null]); + + expect(PriceExperiment::armFor($legacy))->toBe(PriceExperiment::HIGH) + ->and(PriceExperiment::armFor(null))->toBe(PriceExperiment::HIGH); }); it('ignores an invalid forced variant', function () { config(['subscriptions.price_experiment.force_variant' => 'bogus']); - $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-05-01')]); - - expect(PriceExperiment::variantFor($user))->toBe(PriceExperiment::LEGACY); + expect(PriceExperiment::armFor(null))->toBe(PriceExperiment::LEGACY) + ->and(PriceExperiment::isRunning())->toBeTrue(); }); -it('splits post-start users across control and high and stays stable per user', function () { - $variants = []; +it('draws both arms over enough visitors', function () { + $draws = collect(range(1, 60))->map(fn () => PriceExperiment::draw()); - for ($i = 0; $i < 40; $i++) { - $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); - $assigned = PriceExperiment::variantFor($user); - $variants[] = $assigned; - - expect(PriceExperiment::variantFor($user))->toBe($assigned); - } - - expect(array_values(array_unique($variants)))->toEqualCanonicalizing([ + expect($draws->unique()->values()->all())->toEqualCanonicalizing([ PriceExperiment::CONTROL, PriceExperiment::HIGH, ]); }); -it('salts the split so it does not mirror a plain crc32 bucket of the same id', function () { - // An unsalted crc32(id) % 2 would tie this experiment to the buckets of every - // other experiment on the same ids forever. The two splits must disagree. - $disagreements = 0; - - for ($i = 0; $i < 50; $i++) { - $id = (string) Str::uuid(); - $salted = crc32('price:'.$id) % 2; - - if ($salted !== crc32($id) % 2) { - $disagreements++; - } - } - - expect($disagreements)->toBeGreaterThan(0); -}); - -it('applies the variant price and lookup key for a high-price user', function () { - config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]); - $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); +it('applies the variant price and lookup key for a high-arm user', function () { + $user = User::factory()->create(['price_arm' => PriceExperiment::HIGH]); $plans = PriceExperiment::plansFor($user); @@ -108,32 +88,80 @@ it('applies the variant price and lookup key for a high-price user', function () ->and(PriceExperiment::lookupKeyFor($user, 'yearly'))->toBe('whisper_pro_yearly_high'); }); -it('leaves control-price users on the config prices', function () { - config(['subscriptions.price_experiment.force_variant' => PriceExperiment::CONTROL]); - $user = User::factory()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); +it('leaves control-arm users on the config prices', function () { + $user = User::factory()->create(['price_arm' => PriceExperiment::CONTROL]); - $plans = PriceExperiment::plansFor($user); - - expect($plans['monthly']['price'])->toBe(3.99) - ->and($plans['monthly']['stripe_lookup_key'])->toBe('whisper_pro_monthly') + expect(PriceExperiment::plansFor($user)['monthly']['price'])->toBe(3.99) ->and(PriceExperiment::lookupKeyFor($user, 'monthly'))->toBe('whisper_pro_monthly'); }); -it('shows the assigned variant price on the paywall', function () { - config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]); - $user = User::factory()->onboarded()->create(['created_at' => CarbonImmutable::parse('2026-06-10')]); +it('ignores the cookie for a signed-in user, so the arm cannot be edited', function () { + $user = User::factory()->create(['price_arm' => PriceExperiment::HIGH]); - $this->actingAs($user) - ->get(route('subscribe')) + expect(PriceExperiment::armFor($user, PriceExperiment::CONTROL))->toBe(PriceExperiment::HIGH); +}); + +it('quotes an anonymous visitor the price their cookie was drawn into', function () { + $this->withCookie(PriceExperiment::COOKIE, PriceExperiment::HIGH) + ->get('/') ->assertOk() ->assertInertia(fn ($page) => $page - ->component('subscription/paywall') ->where('pricing.plans.monthly.price', 8.99) ->where('pricing.plans.yearly.price', 53.94)); }); -it('charges the assigned variant price id at checkout, not a client-supplied one', function () { - config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]); +it('quotes the control price on the very first visit when the draw lands on control', function () { + // No cookie yet: the middleware draws one and must apply it to this same + // response, since the landing is the first page that quotes a price. + $this->get('/') + ->assertOk() + ->assertCookie(PriceExperiment::COOKIE) + ->assertInertia(fn ($page) => $page + ->where('pricing.plans.monthly.price', fn ($price) => in_array($price, [3.99, 8.99], true))); +}); + +it('does not draw an arm while the experiment is off', function () { + config(['subscriptions.price_experiment.started_at' => null]); + + $this->get('/') + ->assertOk() + ->assertCookieMissing(PriceExperiment::COOKIE) + ->assertInertia(fn ($page) => $page->where('pricing.plans.monthly.price', 3.99)); +}); + +it('keeps the arm the visitor already has instead of redrawing it', function () { + $this->withCookie(PriceExperiment::COOKIE, PriceExperiment::HIGH) + ->get('/') + ->assertOk() + ->assertCookieMissing(PriceExperiment::COOKIE); +}); + +it('freezes the visitor arm onto the user at registration', function () { + $this->withCookie(PriceExperiment::COOKIE, PriceExperiment::HIGH) + ->post(route('register'), [ + 'name' => 'Ada', + 'email' => 'ada@example.com', + 'password' => 'password-1234', + 'password_confirmation' => 'password-1234', + ]); + + expect(User::where('email', 'ada@example.com')->value('price_arm'))->toBe(PriceExperiment::HIGH); +}); + +it('registers without an arm when the visitor arrived with no cookie', function () { + config(['subscriptions.price_experiment.started_at' => null]); + + $this->post(route('register'), [ + 'name' => 'Grace', + 'email' => 'grace@example.com', + 'password' => 'password-1234', + 'password_confirmation' => 'password-1234', + ]); + + expect(User::where('email', 'grace@example.com')->value('price_arm'))->toBeNull(); +}); + +it('charges the stored arm price id at checkout, not a client-supplied one', function () { Cache::put('stripe_price_id:whisper_pro_monthly_high', 'price_high_monthly', now()->addHour()); $checkout = Mockery::mock(Checkout::class); @@ -145,6 +173,7 @@ it('charges the assigned variant price id at checkout, not a client-supplied one $user = Mockery::mock(User::class)->shouldIgnoreMissing(); $user->shouldReceive('hasVerifiedEmail')->andReturn(true); $user->shouldReceive('hasProPlan')->andReturn(false); + $user->shouldReceive('getAttribute')->with('price_arm')->andReturn(PriceExperiment::HIGH); $user->shouldReceive('newSubscription') ->once() ->with('default', 'price_high_monthly') @@ -153,5 +182,8 @@ it('charges the assigned variant price id at checkout, not a client-supplied one $this->withoutMiddleware(HandleInertiaRequests::class); $this->actingAs($user); - $this->get(route('subscribe.checkout', ['plan' => 'monthly']))->assertRedirect(); + // The cookie says control; the stored arm must win. + $this->withCookie(PriceExperiment::COOKIE, PriceExperiment::CONTROL) + ->get(route('subscribe.checkout', ['plan' => 'monthly'])) + ->assertRedirect(); });