From 906e3cc2b466428e7efa4c4c66cb56a068475451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Falc=C3=B3n?= Date: Sat, 13 Jun 2026 23:23:34 +0200 Subject: [PATCH] feat(ai): add weekly AI-suggestions cohort report (#530) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Adds `stats:ai-cohort-report` — a monthly scheduled command that posts a **weekly per-cohort** retention/conversion time series for the onboarding AI suggestions feature to Discord. ## Design (pre/post, directional — not causal) We measure whether shipping AI suggestions moves retention/conversion among the users who can actually receive it. This is an **observational pre/post** readout, deliberately chosen over a randomized A/B holdout (low signup volume; we don't want to withhold the feature). It reports a **correlation**, never a cause. - **Cohort (ITT):** users who imported **≥50 transactions in their first 7 days** (a pre-treatment trait). We do *not* condition on who accepted AI — that would reintroduce self-selection. - **Release anchor `R`:** `MIN(ai_consents.accepted_at)`, planted by self-accepting on deploy. - **Metrics, all fixed-horizon from each user's own signup:** - Retention — active ≥14d (`last_active_at ≥ signup+14d`) - Trial — subscribed ≤14d - Paid — `active` subscription ≤30d - AI-acceptance — funnel-health line (descriptive, not causal) - **Right-censoring:** too-young cohorts render as `pend`, never `0`. - **Surge flagging:** weeks with outlier eligible volume (>2.5× median) are flagged ⚡ so a one-time acquisition spike (e.g. the launch/YouTube surge) isn't misread as an organic trend. Staff/test accounts (incl. the one planting the anchor consent) are excluded via `config('ai_suggestions.report.excluded_emails')`. Output goes to `DISCORD_AI_COHORT_WEBHOOK_URL`, falling back to the shared `DISCORD_WEBHOOK_URL`. ## Honest caveat (baked into every report footer) A one-time launch+YouTube signup surge sits right before release and there's no acquisition-source tracking, so cross-release comparisons mix channels. Read **organic-vs-organic cohort trends over a quarter**, not a month-one before/after diff. ## Deploy steps 1. Self-accept the AI consent in prod to plant `R`. 2. Add your email (+ staff) to `AI_SUGGESTIONS_REPORT_EXCLUDED_EMAILS`. 3. Set `DISCORD_AI_COHORT_WEBHOOK_URL` (or rely on the existing admin webhook). ## Tests `tests/Feature/SendAiCohortReportCommandTest.php` — 7 tests / 25 assertions covering eligibility, retention, trial/paid windows, release anchor + pre/post split, maturity censoring, surge flagging, and Discord delivery (incl. webhook fallback). Pint clean. --- .env.example | 11 + .../Commands/SendAiCohortReportCommand.php | 117 +++++++++ app/Services/Ai/AiCohortReportCollector.php | 213 +++++++++++++++++ config/ai_suggestions.php | 22 ++ config/services.php | 1 + routes/console.php | 1 + .../Feature/SendAiCohortReportCommandTest.php | 224 ++++++++++++++++++ 7 files changed, 589 insertions(+) create mode 100644 app/Console/Commands/SendAiCohortReportCommand.php create mode 100644 app/Services/Ai/AiCohortReportCollector.php create mode 100644 tests/Feature/SendAiCohortReportCommandTest.php diff --git a/.env.example b/.env.example index 7f90f50e..6291949f 100644 --- a/.env.example +++ b/.env.example @@ -138,3 +138,14 @@ AI_SUGGESTIONS_OVERBROAD_FRACTION=0.4 AI_SUGGESTIONS_MIN_TRANSACTIONS=50 AI_SUGGESTIONS_THROTTLE_DAYS=30 AI_SUGGESTIONS_CONSENT_VERSION=1 + +# AI Suggestions cohort report (stats:ai-cohort-report) +AI_SUGGESTIONS_REPORT_WEEKS=16 +# Comma-separated staff/test emails to exclude from cohort metrics (e.g. the +# account that plants the release-anchor consent on deploy). +AI_SUGGESTIONS_REPORT_EXCLUDED_EMAILS= + +# Discord webhooks +DISCORD_WEBHOOK_URL= +# Optional dedicated channel for the AI cohort report; falls back to DISCORD_WEBHOOK_URL. +DISCORD_AI_COHORT_WEBHOOK_URL= diff --git a/app/Console/Commands/SendAiCohortReportCommand.php b/app/Console/Commands/SendAiCohortReportCommand.php new file mode 100644 index 00000000..2dec83b9 --- /dev/null +++ b/app/Console/Commands/SendAiCohortReportCommand.php @@ -0,0 +1,117 @@ +option('weeks') !== null ? (int) $this->option('weeks') : null; + + $report = $this->collector->collect($weeks); + + $webhookUrl = config('services.discord.ai_cohort_webhook_url') + ?: config('services.discord.webhook_url'); + + (new DiscordWebhook($webhookUrl))->send('', [$this->buildEmbed($report)]); + + $this->info('AI cohort report sent to Discord.'); + + return self::SUCCESS; + } + + /** + * @param array{releaseAt: ?CarbonImmutable, releaseWeek: ?string, weeks: list>} $report + * @return array + */ + private function buildEmbed(array $report): array + { + $lines = [sprintf('%-9s %5s %6s %6s %6s %5s', 'Week', 'Elig', 'Ret', 'Trial', 'Paid', 'AI')]; + + foreach ($report['weeks'] as $row) { + $flags = ''; + + if ($report['releaseWeek'] !== null && $row['week'] === $report['releaseWeek']) { + $flags .= ' 🚀'; + } + + if ($row['surge']) { + $flags .= ' ⚡'; + } + + $lines[] = sprintf( + '%-9s %5d %6s %6s %6s %5s%s', + $row['week'], + $row['eligible'], + $this->rateCell($row['retainedRate'], $row['retentionMature'], $row['eligible']), + $this->rateCell($row['trialRate'], $row['retentionMature'], $row['eligible']), + $this->rateCell($row['paidRate'], $row['paidMature'], $row['eligible']), + $this->aiCell($row['aiAcceptedRate'], $row['eligible']), + $flags, + ); + } + + $release = $report['releaseAt'] !== null + ? 'First AI consent (release anchor): '.$report['releaseAt']->format('D, d M Y').' · week '.$report['releaseWeek'].' 🚀' + : 'No AI consent recorded yet — feature not live in production.'; + + return [ + 'title' => '🤖 AI Suggestions — Weekly Cohort Report', + 'description' => "```\n".implode("\n", $lines)."\n```", + 'color' => 0x5865F2, + 'fields' => [ + [ + 'name' => 'Release anchor', + 'value' => $release, + 'inline' => false, + ], + [ + 'name' => 'Legend', + 'value' => 'Elig = users with ≥50 transactions in their first 7 days · Ret = active ≥14d after signup · Trial = subscribed ≤14d · Paid = active subscription ≤30d · AI = accepted AI consent · `pend` = cohort too young to score · ⚡ = signup surge', + 'inline' => false, + ], + [ + 'name' => '⚠️ Directional only', + 'value' => 'Pre/post comparison, not a randomised test. Cohorts are compared at equal age. Surge weeks (⚡, e.g. launch/YouTube) differ in acquisition channel and are not controlled — compare organic weeks like-for-like. Confidence builds over a quarter, not a single month.', + 'inline' => false, + ], + ], + ]; + } + + private function rateCell(?float $rate, bool $mature, int $eligible): string + { + if ($eligible === 0) { + return '—'; + } + + if (! $mature || $rate === null) { + return 'pend'; + } + + return ((int) round($rate * 100)).'%'; + } + + private function aiCell(?float $rate, int $eligible): string + { + if ($eligible === 0 || $rate === null) { + return '—'; + } + + return ((int) round($rate * 100)).'%'; + } +} diff --git a/app/Services/Ai/AiCohortReportCollector.php b/app/Services/Ai/AiCohortReportCollector.php new file mode 100644 index 00000000..5006759b --- /dev/null +++ b/app/Services/Ai/AiCohortReportCollector.php @@ -0,0 +1,213 @@ + + * } + */ + public function collect(?int $weeks = null): array + { + $weeks = max(1, $weeks ?? (int) config('ai_suggestions.report.weeks', self::DEFAULT_WEEKS)); + + $now = CarbonImmutable::now('UTC'); + $windowStart = $now->startOfWeek(CarbonImmutable::MONDAY)->subWeeks($weeks - 1); + + $releaseValue = AiConsent::query()->min('accepted_at'); + $releaseAt = $releaseValue !== null ? CarbonImmutable::parse($releaseValue) : null; + $releaseWeekStart = $releaseAt?->startOfWeek(CarbonImmutable::MONDAY); + + $aggregates = $this->aggregateEligibleUsers($windowStart); + + $rows = []; + $eligibleCounts = []; + + for ($i = 0; $i < $weeks; $i++) { + $weekStart = $windowStart->addWeeks($i); + $weekEnd = $weekStart->endOfWeek(CarbonImmutable::SUNDAY); + $key = (int) $weekStart->format('oW'); + + $agg = $aggregates[$key] ?? [ + 'eligible' => 0, + 'retained' => 0, + 'trial' => 0, + 'paid' => 0, + 'aiAccepted' => 0, + ]; + + $eligible = $agg['eligible']; + $retentionMature = $weekEnd->addDays(self::RETENTION_DAYS)->lessThanOrEqualTo($now); + $paidMature = $weekEnd->addDays(self::PAID_DAYS)->lessThanOrEqualTo($now); + + $eligibleCounts[] = $eligible; + + $rows[$i] = [ + 'week' => $weekStart->format('o-\WW'), + 'weekStart' => $weekStart, + 'eligible' => $eligible, + 'retained' => $agg['retained'], + 'retainedRate' => $retentionMature && $eligible > 0 ? $agg['retained'] / $eligible : null, + 'trial' => $agg['trial'], + 'trialRate' => $retentionMature && $eligible > 0 ? $agg['trial'] / $eligible : null, + 'paid' => $agg['paid'], + 'paidRate' => $paidMature && $eligible > 0 ? $agg['paid'] / $eligible : null, + 'aiAccepted' => $agg['aiAccepted'], + 'aiAcceptedRate' => $eligible > 0 ? $agg['aiAccepted'] / $eligible : null, + 'phase' => $this->phase($weekStart, $releaseWeekStart), + 'surge' => false, + 'retentionMature' => $retentionMature, + 'paidMature' => $paidMature, + ]; + } + + $this->flagSurges($rows, $eligibleCounts); + + return [ + 'releaseAt' => $releaseAt, + 'releaseWeek' => $releaseWeekStart?->format('o-\WW'), + 'weeks' => array_values($rows), + ]; + } + + /** + * Aggregate per-user metric flags for eligible users, keyed by ISO year-week. + * + * @return array + */ + private function aggregateEligibleUsers(CarbonImmutable $windowStart): array + { + $threshold = (int) config('ai_suggestions.eligibility_min_transactions', 50); + $excluded = (array) config('ai_suggestions.report.excluded_emails', []); + + $rows = User::query() + ->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded)) + ->where('users.created_at', '>=', $windowStart) + ->whereHas('transactions', function ($query): void { + $query->whereRaw( + 'transactions.created_at <= DATE_ADD(users.created_at, INTERVAL '.self::ELIGIBILITY_WINDOW_DAYS.' DAY)', + ); + }, '>=', $threshold) + ->selectRaw('YEARWEEK(users.created_at, 3) as yearweek') + ->selectRaw('(users.last_active_at IS NOT NULL AND users.last_active_at >= DATE_ADD(users.created_at, INTERVAL '.self::RETENTION_DAYS.' DAY)) as retained') + ->selectRaw('EXISTS(SELECT 1 FROM subscriptions s WHERE s.user_id = users.id AND s.created_at <= DATE_ADD(users.created_at, INTERVAL '.self::TRIAL_DAYS.' DAY)) as has_trial') + ->selectRaw("EXISTS(SELECT 1 FROM subscriptions s WHERE s.user_id = users.id AND s.stripe_status = 'active' AND s.created_at <= DATE_ADD(users.created_at, INTERVAL ".self::PAID_DAYS.' DAY)) as has_paid') + ->selectRaw('EXISTS(SELECT 1 FROM ai_consents c WHERE c.user_id = users.id AND c.accepted_at IS NOT NULL) as ai_accepted') + ->toBase() + ->get(); + + $aggregates = []; + + foreach ($rows as $row) { + $key = (int) $row->yearweek; + + if (! isset($aggregates[$key])) { + $aggregates[$key] = ['eligible' => 0, 'retained' => 0, 'trial' => 0, 'paid' => 0, 'aiAccepted' => 0]; + } + + $aggregates[$key]['eligible']++; + $aggregates[$key]['retained'] += (int) $row->retained; + $aggregates[$key]['trial'] += (int) $row->has_trial; + $aggregates[$key]['paid'] += (int) $row->has_paid; + $aggregates[$key]['aiAccepted'] += (int) $row->ai_accepted; + } + + return $aggregates; + } + + private function phase(CarbonImmutable $weekStart, ?CarbonImmutable $releaseWeekStart): string + { + if ($releaseWeekStart === null) { + return 'pre'; + } + + return $weekStart->lessThan($releaseWeekStart) ? 'pre' : 'post'; + } + + /** + * Flag weeks whose eligible volume is an outlier (e.g. a launch/marketing + * spike) so a non-representative acquisition wave can't be read as an + * organic trend. + * + * @param array> $rows + * @param list $eligibleCounts + */ + private function flagSurges(array &$rows, array $eligibleCounts): void + { + $nonZero = array_values(array_filter($eligibleCounts, fn (int $count): bool => $count > 0)); + $median = $this->median($nonZero); + + if ($median <= 0.0) { + return; + } + + foreach ($rows as $index => $row) { + if ($row['eligible'] > self::SURGE_MULTIPLIER * $median) { + $rows[$index]['surge'] = true; + } + } + } + + /** + * @param list $values + */ + private function median(array $values): float + { + sort($values); + $count = count($values); + + if ($count === 0) { + return 0.0; + } + + $middle = intdiv($count, 2); + + if ($count % 2 === 1) { + return (float) $values[$middle]; + } + + return ($values[$middle - 1] + $values[$middle]) / 2; + } +} diff --git a/config/ai_suggestions.php b/config/ai_suggestions.php index efb23f3d..57cf4f79 100644 --- a/config/ai_suggestions.php +++ b/config/ai_suggestions.php @@ -108,4 +108,26 @@ return [ 'consent_version' => (string) env('AI_SUGGESTIONS_CONSENT_VERSION', '1'), + /* + |-------------------------------------------------------------------------- + | Cohort Report + |-------------------------------------------------------------------------- + | + | Settings for the weekly AI-suggestions cohort report (the + | `stats:ai-cohort-report` command). `weeks` is how many weekly cohorts to + | include; `excluded_emails` is a comma-separated list of staff/test + | accounts (e.g. the user who plants the release-anchor consent) that must + | be kept out of the cohort metrics. + | + */ + + 'report' => [ + 'weeks' => (int) env('AI_SUGGESTIONS_REPORT_WEEKS', 16), + + 'excluded_emails' => array_values(array_filter(array_map( + 'trim', + explode(',', (string) env('AI_SUGGESTIONS_REPORT_EXCLUDED_EMAILS', '')), + ))), + ], + ]; diff --git a/config/services.php b/config/services.php index a9d48fe8..78a201ba 100644 --- a/config/services.php +++ b/config/services.php @@ -45,6 +45,7 @@ return [ 'discord' => [ 'webhook_url' => env('DISCORD_WEBHOOK_URL'), + 'ai_cohort_webhook_url' => env('DISCORD_AI_COHORT_WEBHOOK_URL'), ], ]; diff --git a/routes/console.php b/routes/console.php index 970c55af..a9fc657e 100644 --- a/routes/console.php +++ b/routes/console.php @@ -12,3 +12,4 @@ Schedule::command('resend:sync-leads')->dailyAt('03:00'); Schedule::command('leads:send-invitations --force --limit=150')->dailyAt('09:00'); Schedule::command('leads:send-re-invitations --force')->dailyAt('09:00'); Schedule::command('stats:daily-report')->dailyAt('09:00')->timezone('Europe/Madrid'); +Schedule::command('stats:ai-cohort-report')->monthlyOn(1, '09:00')->timezone('Europe/Madrid'); diff --git a/tests/Feature/SendAiCohortReportCommandTest.php b/tests/Feature/SendAiCohortReportCommandTest.php new file mode 100644 index 00000000..7ea5cc98 --- /dev/null +++ b/tests/Feature/SendAiCohortReportCommandTest.php @@ -0,0 +1,224 @@ + $opts + */ +function cohortUser(CarbonImmutable $signup, array $opts = []): User +{ + $user = User::factory()->create([ + 'email' => $opts['email'] ?? fake()->unique()->safeEmail(), + 'created_at' => $signup, + 'last_active_at' => $opts['lastActiveAt'] ?? null, + ]); + + $account = Account::factory()->for($user)->create(); + $category = Category::factory()->for($user)->create(); + + Transaction::factory()->count($opts['transactions'] ?? 3)->for($user)->for($account)->create([ + 'category_id' => $category->id, + 'created_at' => $signup->addDay(), + ]); + + if (! empty($opts['consentAt'])) { + AiConsent::factory()->for($user)->create(['accepted_at' => $opts['consentAt']]); + } + + if (! empty($opts['subscription'])) { + $user->subscriptions()->create([ + 'type' => 'default', + 'stripe_id' => 'sub_'.Str::random(12), + 'stripe_status' => $opts['subscription']['status'], + 'stripe_price' => 'price_test', + 'created_at' => $opts['subscription']['at'], + ]); + } + + return $user; +} + +/** + * @param array{weeks: list>} $report + * @return array + */ +function rowForWeek(array $report, CarbonImmutable $signup): array +{ + $label = $signup->startOfWeek(CarbonImmutable::MONDAY)->format('o-\WW'); + + foreach ($report['weeks'] as $row) { + if ($row['week'] === $label) { + return $row; + } + } + + throw new RuntimeException("No cohort row found for week {$label}"); +} + +beforeEach(function () { + Carbon::setTestNow(referenceNow()); + config(['ai_suggestions.eligibility_min_transactions' => 3]); + config(['ai_suggestions.report.excluded_emails' => []]); +}); + +it('only counts users who imported enough transactions within their first week', function () { + $signup = referenceNow()->subWeeks(6); + + cohortUser($signup, ['transactions' => 3, 'lastActiveAt' => $signup->addDays(20)]); // eligible, retained + cohortUser($signup, ['transactions' => 3, 'lastActiveAt' => $signup->addDays(2)]); // eligible, churned early + cohortUser($signup, ['transactions' => 2, 'lastActiveAt' => $signup->addDays(20)]); // ineligible (<3 txns) + + // 3 transactions, but imported on day 10 — outside the 7-day eligibility window. + $late = cohortUser($signup, ['transactions' => 0, 'lastActiveAt' => $signup->addDays(20)]); + $account = Account::factory()->for($late)->create(); + Transaction::factory()->count(3)->for($late)->for($account)->create(['created_at' => $signup->addDays(10)]); + + $report = app(AiCohortReportCollector::class)->collect(); + $row = rowForWeek($report, $signup); + + expect($row['eligible'])->toBe(2) + ->and($row['retained'])->toBe(1) + ->and($row['retainedRate'])->toBe(0.5) + ->and($row['retentionMature'])->toBeTrue(); +}); + +it('counts trial starts within 14 days and active paid conversions within 30 days', function () { + $signup = referenceNow()->subWeeks(6); + + // Trialing subscription started at signup: counts as trial, not as paid. + cohortUser($signup, [ + 'transactions' => 3, + 'subscription' => ['status' => 'trialing', 'at' => $signup->addDays(2)], + ]); + + // Active subscription within 30 days: counts as both trial and paid. + cohortUser($signup, [ + 'transactions' => 3, + 'subscription' => ['status' => 'active', 'at' => $signup->addDays(5)], + ]); + + // Active subscription, but only after day 40: outside both windows. + cohortUser($signup, [ + 'transactions' => 3, + 'subscription' => ['status' => 'active', 'at' => $signup->addDays(40)], + ]); + + $report = app(AiCohortReportCollector::class)->collect(); + $row = rowForWeek($report, $signup); + + expect($row['eligible'])->toBe(3) + ->and($row['trial'])->toBe(2) + ->and($row['paid'])->toBe(1) + ->and($row['paidMature'])->toBeTrue(); +}); + +it('derives the release anchor from the first consent and splits cohorts pre/post', function () { + config(['ai_suggestions.report.excluded_emails' => ['staff@whisper.test']]); + + $release = referenceNow()->subWeeks(4); + + // Staff account plants the release-anchor consent; excluded from cohort metrics. + cohortUser($release, [ + 'email' => 'staff@whisper.test', + 'transactions' => 3, + 'consentAt' => $release, + ]); + + $before = referenceNow()->subWeeks(6); + $after = referenceNow()->subWeeks(2); + cohortUser($before, ['transactions' => 3]); + cohortUser($after, ['transactions' => 3]); + + $report = app(AiCohortReportCollector::class)->collect(); + + expect($report['releaseWeek'])->toBe($release->startOfWeek(CarbonImmutable::MONDAY)->format('o-\WW')) + ->and(rowForWeek($report, $before)['phase'])->toBe('pre') + ->and(rowForWeek($report, $after)['phase'])->toBe('post') + // Staff account was excluded, so the release week has no eligible users. + ->and(rowForWeek($report, $release)['eligible'])->toBe(0); +}); + +it('marks cohorts as not mature until the metric horizon has elapsed', function () { + $recent = referenceNow()->subDays(3); // current week — too young for any 14d metric + $midAged = referenceNow()->subWeeks(3); // older than 14d, younger than 30d + + cohortUser($recent, ['transactions' => 3, 'lastActiveAt' => $recent->addDay()]); + cohortUser($midAged, ['transactions' => 3, 'lastActiveAt' => $midAged->addDays(20)]); + + $report = app(AiCohortReportCollector::class)->collect(); + + $recentRow = rowForWeek($report, $recent); + expect($recentRow['retentionMature'])->toBeFalse() + ->and($recentRow['retainedRate'])->toBeNull() + ->and($recentRow['paidRate'])->toBeNull(); + + $midRow = rowForWeek($report, $midAged); + expect($midRow['retentionMature'])->toBeTrue() + ->and($midRow['retainedRate'])->not->toBeNull() + ->and($midRow['paidMature'])->toBeFalse() + ->and($midRow['paidRate'])->toBeNull(); +}); + +it('flags weeks whose eligible volume is an outlier surge', function () { + $surgeWeek = referenceNow()->subWeeks(5); + $quietWeeks = [referenceNow()->subWeeks(8), referenceNow()->subWeeks(7), referenceNow()->subWeeks(6)]; + + foreach ($quietWeeks as $week) { + cohortUser($week, ['transactions' => 3]); + } + + for ($i = 0; $i < 6; $i++) { + cohortUser($surgeWeek, ['transactions' => 3]); + } + + $report = app(AiCohortReportCollector::class)->collect(); + + expect(rowForWeek($report, $surgeWeek)['surge'])->toBeTrue() + ->and(rowForWeek($report, $quietWeeks[0])['surge'])->toBeFalse(); +}); + +it('posts the cohort report embed to the configured discord webhook', function () { + config(['services.discord.ai_cohort_webhook_url' => 'https://discord.test/hook']); + Http::fake(['discord.test/*' => Http::response('', 204)]); + + cohortUser(referenceNow()->subWeeks(6), ['transactions' => 3, 'lastActiveAt' => referenceNow()->subWeeks(3)]); + + artisan('stats:ai-cohort-report')->assertSuccessful(); + + Http::assertSent(function ($request) { + return $request->url() === 'https://discord.test/hook' + && isset($request['embeds'][0]['title']) + && str_contains($request['embeds'][0]['title'], 'AI Suggestions'); + }); +}); + +it('falls back to the default discord webhook when no dedicated one is set', function () { + config([ + 'services.discord.ai_cohort_webhook_url' => null, + 'services.discord.webhook_url' => 'https://discord.test/default', + ]); + Http::fake(['discord.test/*' => Http::response('', 204)]); + + artisan('stats:ai-cohort-report')->assertSuccessful(); + + Http::assertSent(fn ($request) => $request->url() === 'https://discord.test/default'); +});