diff --git a/app/Console/Commands/SendExperimentFunnelReportCommand.php b/app/Console/Commands/SendExperimentFunnelReportCommand.php new file mode 100644 index 00000000..32922106 --- /dev/null +++ b/app/Console/Commands/SendExperimentFunnelReportCommand.php @@ -0,0 +1,111 @@ + 'control', + SubscriptionExperiment::REDUCED_TRIAL => 'reduced', + SubscriptionExperiment::PAY_NOW => 'pay_now', + ]; + + public function __construct(private ExperimentFunnelCollector $collector) + { + parent::__construct(); + } + + public function handle(): int + { + $report = $this->collector->collect(); + + if ($report['startedAt'] === null) { + $this->warn('Experiment not started — set SUBSCRIPTION_EXPERIMENT_STARTED_AT to begin.'); + + return self::SUCCESS; + } + + foreach ($this->tableLines($report) as $line) { + $this->line($line); + } + + $webhookUrl = config('services.discord.ai_cohort_webhook_url') + ?: config('services.discord.webhook_url'); + + (new DiscordWebhook($webhookUrl))->send('', [$this->buildEmbed($report)]); + + $this->info('Experiment funnel report sent to Discord.'); + + return self::SUCCESS; + } + + /** + * @param array{startedAt: ?CarbonImmutable, variants: array>} $report + * @return list + */ + private function tableLines(array $report): array + { + $lines = [sprintf('%-8s %5s %4s %5s %5s %5s %5s %5s %5s', 'Variant', 'Assg', 'Sub', 'Trial', 'Actv', 'Cncl', 'PstD', 'Rfnd', 'Net%')]; + + foreach (self::LABELS as $key => $label) { + $row = $report['variants'][$key]; + + $lines[] = sprintf( + '%-8s %5d %4d %5d %5d %5d %5d %5d %5s', + $label, + $row['assigned'], + $row['subscribed'], + $row['trialing'], + $row['active'], + $row['canceled'], + $row['pastDue'], + $row['refunded'], + $row['assignedMature'] === 0 + ? 'pend' + : ((int) round($row['netActiveRate'] * 100)).'%', + ); + } + + return $lines; + } + + /** + * @param array{startedAt: ?CarbonImmutable, variants: array>} $report + * @return array + */ + private function buildEmbed(array $report): array + { + return [ + 'title' => '🧪 Trial/Pricing Experiment — Funnel by Variant', + 'description' => "```\n".implode("\n", $this->tableLines($report))."\n```", + 'color' => 0xFEE75C, + 'fields' => [ + [ + 'name' => 'Started', + 'value' => $report['startedAt']->format('D, d M Y').' · new signups split evenly into the three variants.', + 'inline' => false, + ], + [ + 'name' => 'Legend', + 'value' => 'Assg = assigned to the variant · Sub = started a plan · Trial/Actv/Cncl/PstD = current subscription status · Rfnd = self-service refunds (pay_now) · Net% = live, non-refunded subscriptions ÷ assigned, counting only users past their decision window · `pend` = no mature users yet.', + 'inline' => false, + ], + [ + 'name' => '⚠️ Read at equal age', + 'value' => 'Net% gates each variant by its own decision window (control 15d, reduced 7d, pay_now 3d), so pay_now matures first. Compare Net% only once all three have meaningful mature volume.', + 'inline' => false, + ], + ], + ]; + } +} diff --git a/app/Features/SubscriptionExperiment.php b/app/Features/SubscriptionExperiment.php index cc496db2..a2c1c3e2 100644 --- a/app/Features/SubscriptionExperiment.php +++ b/app/Features/SubscriptionExperiment.php @@ -37,7 +37,17 @@ class SubscriptionExperiment return self::LEGACY; } - return match (crc32((string) $user->getKey()) % 3) { + return self::bucket((string) $user->getKey()); + } + + /** + * Deterministic, evenly-split bucket for a post-start user. The funnel report + * mirrors this in PHP to attribute users without reading Pennant per row, so + * keep the formula here as the single source of truth. + */ + public static function bucket(string $key): string + { + return match (crc32($key) % 3) { 0 => self::CONTROL, 1 => self::REDUCED_TRIAL, default => self::PAY_NOW, diff --git a/app/Services/Stats/ExperimentFunnelCollector.php b/app/Services/Stats/ExperimentFunnelCollector.php new file mode 100644 index 00000000..e9036413 --- /dev/null +++ b/app/Services/Stats/ExperimentFunnelCollector.php @@ -0,0 +1,142 @@ + + * } + */ + public function collect(): array + { + $startedValue = config('subscriptions.experiment.started_at'); + $startedAt = $startedValue !== null ? CarbonImmutable::parse($startedValue) : null; + + $variants = [ + SubscriptionExperiment::CONTROL => $this->emptyRow(), + SubscriptionExperiment::REDUCED_TRIAL => $this->emptyRow(), + SubscriptionExperiment::PAY_NOW => $this->emptyRow(), + ]; + + if ($startedAt === null) { + return ['startedAt' => null, 'variants' => $variants]; + } + + $now = CarbonImmutable::now('UTC'); + $excluded = (array) config('ai_suggestions.report.excluded_emails', []); + $windows = $this->decisionWindows(); + + User::query() + ->where('users.created_at', '>=', $startedAt) + ->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded)) + ->with(['subscriptions' => fn ($query) => $query->where('type', 'default')]) + ->select(['id', 'created_at']) + ->chunkById(500, function ($users) use (&$variants, $windows, $now): void { + foreach ($users as $user) { + $variant = SubscriptionExperiment::bucket((string) $user->getKey()); + $row = &$variants[$variant]; + + $row['assigned']++; + + $subscription = $user->subscriptions->sortByDesc('created_at')->first(); + $status = $subscription?->stripe_status; + $netActive = $status === 'active' && $subscription->refunded_at === null; + + if ($subscription !== null) { + $row['subscribed']++; + $row['trialing'] += $status === 'trialing' ? 1 : 0; + $row['active'] += $status === 'active' ? 1 : 0; + $row['canceled'] += $status === 'canceled' ? 1 : 0; + $row['pastDue'] += $status === 'past_due' ? 1 : 0; + $row['refunded'] += $subscription->refunded_at !== null ? 1 : 0; + } + + $mature = CarbonImmutable::parse($user->created_at) + ->addDays($windows[$variant] + self::SETTLE_BUFFER_DAYS) + ->lessThanOrEqualTo($now); + + if ($mature) { + $row['assignedMature']++; + $row['activeMature'] += $netActive ? 1 : 0; + } + + unset($row); + } + }); + + foreach ($variants as $key => $row) { + $variants[$key]['netActiveRate'] = $row['assignedMature'] > 0 + ? $row['activeMature'] / $row['assignedMature'] + : null; + } + + return ['startedAt' => $startedAt, 'variants' => $variants]; + } + + /** + * @return array{assigned: int, subscribed: int, trialing: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activeMature: int, netActiveRate: ?float} + */ + private function emptyRow(): array + { + return [ + 'assigned' => 0, + 'subscribed' => 0, + 'trialing' => 0, + 'active' => 0, + 'canceled' => 0, + 'pastDue' => 0, + 'refunded' => 0, + 'assignedMature' => 0, + 'activeMature' => 0, + 'netActiveRate' => null, + ]; + } + + /** + * Days from signup until each variant's outcome can be scored: the trial + * length (the longer of the two reduced trials) or the refund window. + * + * @return array + */ + private function decisionWindows(): array + { + return [ + SubscriptionExperiment::CONTROL => (int) config('subscriptions.plans.monthly.trial_days', 15), + SubscriptionExperiment::REDUCED_TRIAL => max( + (int) config('subscriptions.experiment.reduced_trial.monthly', 3), + (int) config('subscriptions.experiment.reduced_trial.yearly', 7), + ), + SubscriptionExperiment::PAY_NOW => (int) config('subscriptions.experiment.pay_now_refund_window_days', 3), + ]; + } +} diff --git a/routes/console.php b/routes/console.php index 3f324e9a..7c630c3a 100644 --- a/routes/console.php +++ b/routes/console.php @@ -17,3 +17,4 @@ Schedule::command('stats:daily-report')->dailyAt('09:00')->timezone('Europe/Madr Schedule::command('stats:ai-cohort-report')->monthlyOn(1, '09:00')->timezone('Europe/Madrid'); Schedule::command('stats:stuck-cohort-report')->weekly()->mondays()->at('09:00')->timezone('Europe/Madrid'); Schedule::command('stats:subscription-funnel')->weekly()->mondays()->at('09:15')->timezone('Europe/Madrid'); +Schedule::command('stats:experiment-funnel')->weekly()->mondays()->at('09:30')->timezone('Europe/Madrid'); diff --git a/tests/Feature/SendExperimentFunnelReportCommandTest.php b/tests/Feature/SendExperimentFunnelReportCommandTest.php new file mode 100644 index 00000000..2cadf97b --- /dev/null +++ b/tests/Feature/SendExperimentFunnelReportCommandTest.php @@ -0,0 +1,129 @@ + true, + 'subscriptions.experiment.started_at' => '2026-06-01', + 'subscriptions.experiment.reduced_trial.monthly' => 3, + 'subscriptions.experiment.reduced_trial.yearly' => 7, + 'subscriptions.experiment.pay_now_refund_window_days' => 3, + 'subscriptions.plans.monthly.trial_days' => 15, + 'ai_suggestions.report.excluded_emails' => [], + ]); + Carbon::setTestNow(CarbonImmutable::parse('2026-06-30 12:00:00')); +}); + +/** + * Create a user whose id buckets into the wanted variant, anchored to a signup, + * with an optional default subscription. + * + * @param array{status: string, at: CarbonImmutable, endsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription + */ +function experimentUser(string $variant, CarbonImmutable $signup, ?array $subscription = null): User +{ + do { + $id = (string) Str::uuid(); + } while (SubscriptionExperiment::bucket($id) !== $variant); + + $user = User::factory()->create(['id' => $id, 'created_at' => $signup]); + + if ($subscription !== null) { + $user->subscriptions()->create([ + 'type' => 'default', + 'stripe_id' => 'sub_'.Str::random(12), + 'stripe_status' => $subscription['status'], + 'stripe_price' => 'price_test', + 'created_at' => $subscription['at'], + 'ends_at' => $subscription['endsAt'] ?? null, + 'refunded_at' => $subscription['refundedAt'] ?? null, + ]); + } + + return $user; +} + +it('returns empty variants when the experiment has not started', function () { + config(['subscriptions.experiment.started_at' => null]); + + $report = app(ExperimentFunnelCollector::class)->collect(); + + expect($report['startedAt'])->toBeNull() + ->and($report['variants'][SubscriptionExperiment::CONTROL]['assigned'])->toBe(0); +}); + +it('attributes users and their subscription status to the right variant', function () { + $signup = CarbonImmutable::parse('2026-06-05'); // paid-mature for every variant by the test clock + + experimentUser(SubscriptionExperiment::CONTROL, $signup); // assigned, no sub + experimentUser(SubscriptionExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay()]); + experimentUser(SubscriptionExperiment::REDUCED_TRIAL, $signup, ['status' => 'active', 'at' => $signup->addDay()]); + experimentUser(SubscriptionExperiment::PAY_NOW, $signup, ['status' => 'active', 'at' => $signup]); + // pay_now refund: active charge that was refunded -> canceled and not counted as net active. + experimentUser(SubscriptionExperiment::PAY_NOW, $signup, [ + 'status' => 'canceled', + 'at' => $signup, + 'endsAt' => $signup->addDay(), + 'refundedAt' => $signup->addDay(), + ]); + + $variants = app(ExperimentFunnelCollector::class)->collect()['variants']; + + expect($variants[SubscriptionExperiment::CONTROL]['assigned'])->toBe(2) + ->and($variants[SubscriptionExperiment::CONTROL]['subscribed'])->toBe(1) + ->and($variants[SubscriptionExperiment::CONTROL]['active'])->toBe(1) + ->and($variants[SubscriptionExperiment::CONTROL]['netActiveRate'])->toBe(0.5) + ->and($variants[SubscriptionExperiment::REDUCED_TRIAL]['active'])->toBe(1) + ->and($variants[SubscriptionExperiment::PAY_NOW]['assigned'])->toBe(2) + ->and($variants[SubscriptionExperiment::PAY_NOW]['active'])->toBe(1) + ->and($variants[SubscriptionExperiment::PAY_NOW]['refunded'])->toBe(1) + ->and($variants[SubscriptionExperiment::PAY_NOW]['activeMature'])->toBe(1); +}); + +it('leaves young cohorts out of the mature net-active rate', function () { + // pay_now decides in 3d (+3 buffer); a 2-day-old signup is not mature yet. + experimentUser(SubscriptionExperiment::PAY_NOW, CarbonImmutable::now()->subDays(2), [ + 'status' => 'active', + 'at' => CarbonImmutable::now()->subDays(2), + ]); + + $payNow = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::PAY_NOW]; + + expect($payNow['assigned'])->toBe(1) + ->and($payNow['active'])->toBe(1) + ->and($payNow['assignedMature'])->toBe(0) + ->and($payNow['netActiveRate'])->toBeNull(); +}); + +it('posts the experiment funnel embed to discord', function () { + config(['services.discord.ai_cohort_webhook_url' => 'https://discord.test/hook']); + Http::fake(['discord.test/*' => Http::response('', 204)]); + + experimentUser(SubscriptionExperiment::CONTROL, CarbonImmutable::parse('2026-06-05'), [ + 'status' => 'active', + 'at' => CarbonImmutable::parse('2026-06-05'), + ]); + + artisan('stats:experiment-funnel')->assertSuccessful(); + + Http::assertSent(fn ($request) => $request->url() === 'https://discord.test/hook' + && str_contains($request['embeds'][0]['title'], 'Experiment')); +}); + +it('does not post when the experiment has not started', function () { + config(['subscriptions.experiment.started_at' => null]); + Http::fake(); + + artisan('stats:experiment-funnel')->assertSuccessful(); + + Http::assertNothingSent(); +});