feat(ai): add weekly AI-suggestions cohort report (#530)
## What
Adds `stats:ai-cohort-report` — a monthly scheduled command that posts a
**weekly per-cohort** retention/conversion time series for the
onboarding AI suggestions feature to Discord.
## Design (pre/post, directional — not causal)
We measure whether shipping AI suggestions moves retention/conversion
among the users who can actually receive it. This is an **observational
pre/post** readout, deliberately chosen over a randomized A/B holdout
(low signup volume; we don't want to withhold the feature). It reports a
**correlation**, never a cause.
- **Cohort (ITT):** users who imported **≥50 transactions in their first
7 days** (a pre-treatment trait). We do *not* condition on who accepted
AI — that would reintroduce self-selection.
- **Release anchor `R`:** `MIN(ai_consents.accepted_at)`, planted by
self-accepting on deploy.
- **Metrics, all fixed-horizon from each user's own signup:**
- Retention — active ≥14d (`last_active_at ≥ signup+14d`)
- Trial — subscribed ≤14d
- Paid — `active` subscription ≤30d
- AI-acceptance — funnel-health line (descriptive, not causal)
- **Right-censoring:** too-young cohorts render as `pend`, never `0`.
- **Surge flagging:** weeks with outlier eligible volume (>2.5× median)
are flagged ⚡ so a one-time acquisition spike (e.g. the launch/YouTube
surge) isn't misread as an organic trend.
Staff/test accounts (incl. the one planting the anchor consent) are
excluded via `config('ai_suggestions.report.excluded_emails')`. Output
goes to `DISCORD_AI_COHORT_WEBHOOK_URL`, falling back to the shared
`DISCORD_WEBHOOK_URL`.
## Honest caveat (baked into every report footer)
A one-time launch+YouTube signup surge sits right before release and
there's no acquisition-source tracking, so cross-release comparisons mix
channels. Read **organic-vs-organic cohort trends over a quarter**, not
a month-one before/after diff.
## Deploy steps
1. Self-accept the AI consent in prod to plant `R`.
2. Add your email (+ staff) to `AI_SUGGESTIONS_REPORT_EXCLUDED_EMAILS`.
3. Set `DISCORD_AI_COHORT_WEBHOOK_URL` (or rely on the existing admin
webhook).
## Tests
`tests/Feature/SendAiCohortReportCommandTest.php` — 7 tests / 25
assertions covering eligibility, retention, trial/paid windows, release
anchor + pre/post split, maturity censoring, surge flagging, and Discord
delivery (incl. webhook fallback). Pint clean.
This commit is contained in:
Víctor Falcón
GitHub
parent
8056ede636
commit
906e3cc2b4
11
.env.example
11
.env.example
|
|
@ -138,3 +138,14 @@ AI_SUGGESTIONS_OVERBROAD_FRACTION=0.4
|
|||
AI_SUGGESTIONS_MIN_TRANSACTIONS=50
|
||||
AI_SUGGESTIONS_THROTTLE_DAYS=30
|
||||
AI_SUGGESTIONS_CONSENT_VERSION=1
|
||||
|
||||
# AI Suggestions cohort report (stats:ai-cohort-report)
|
||||
AI_SUGGESTIONS_REPORT_WEEKS=16
|
||||
# Comma-separated staff/test emails to exclude from cohort metrics (e.g. the
|
||||
# account that plants the release-anchor consent on deploy).
|
||||
AI_SUGGESTIONS_REPORT_EXCLUDED_EMAILS=
|
||||
|
||||
# Discord webhooks
|
||||
DISCORD_WEBHOOK_URL=
|
||||
# Optional dedicated channel for the AI cohort report; falls back to DISCORD_WEBHOOK_URL.
|
||||
DISCORD_AI_COHORT_WEBHOOK_URL=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Services\Ai\AiCohortReportCollector;
|
||||
use App\Services\Discord\DiscordWebhook;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class SendAiCohortReportCommand extends Command
|
||||
{
|
||||
protected $signature = 'stats:ai-cohort-report {--weeks= : Number of weekly cohorts to include}';
|
||||
|
||||
protected $description = 'Post the weekly AI-suggestions cohort retention/conversion report to Discord';
|
||||
|
||||
public function __construct(private AiCohortReportCollector $collector)
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
public function handle(): int
|
||||
{
|
||||
$weeks = $this->option('weeks') !== null ? (int) $this->option('weeks') : null;
|
||||
|
||||
$report = $this->collector->collect($weeks);
|
||||
|
||||
$webhookUrl = config('services.discord.ai_cohort_webhook_url')
|
||||
?: config('services.discord.webhook_url');
|
||||
|
||||
(new DiscordWebhook($webhookUrl))->send('', [$this->buildEmbed($report)]);
|
||||
|
||||
$this->info('AI cohort report sent to Discord.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{releaseAt: ?CarbonImmutable, releaseWeek: ?string, weeks: list<array<string, mixed>>} $report
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function buildEmbed(array $report): array
|
||||
{
|
||||
$lines = [sprintf('%-9s %5s %6s %6s %6s %5s', 'Week', 'Elig', 'Ret', 'Trial', 'Paid', 'AI')];
|
||||
|
||||
foreach ($report['weeks'] as $row) {
|
||||
$flags = '';
|
||||
|
||||
if ($report['releaseWeek'] !== null && $row['week'] === $report['releaseWeek']) {
|
||||
$flags .= ' 🚀';
|
||||
}
|
||||
|
||||
if ($row['surge']) {
|
||||
$flags .= ' ⚡';
|
||||
}
|
||||
|
||||
$lines[] = sprintf(
|
||||
'%-9s %5d %6s %6s %6s %5s%s',
|
||||
$row['week'],
|
||||
$row['eligible'],
|
||||
$this->rateCell($row['retainedRate'], $row['retentionMature'], $row['eligible']),
|
||||
$this->rateCell($row['trialRate'], $row['retentionMature'], $row['eligible']),
|
||||
$this->rateCell($row['paidRate'], $row['paidMature'], $row['eligible']),
|
||||
$this->aiCell($row['aiAcceptedRate'], $row['eligible']),
|
||||
$flags,
|
||||
);
|
||||
}
|
||||
|
||||
$release = $report['releaseAt'] !== null
|
||||
? 'First AI consent (release anchor): '.$report['releaseAt']->format('D, d M Y').' · week '.$report['releaseWeek'].' 🚀'
|
||||
: 'No AI consent recorded yet — feature not live in production.';
|
||||
|
||||
return [
|
||||
'title' => '🤖 AI Suggestions — Weekly Cohort Report',
|
||||
'description' => "```\n".implode("\n", $lines)."\n```",
|
||||
'color' => 0x5865F2,
|
||||
'fields' => [
|
||||
[
|
||||
'name' => 'Release anchor',
|
||||
'value' => $release,
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => 'Legend',
|
||||
'value' => 'Elig = users with ≥50 transactions in their first 7 days · Ret = active ≥14d after signup · Trial = subscribed ≤14d · Paid = active subscription ≤30d · AI = accepted AI consent · `pend` = cohort too young to score · ⚡ = signup surge',
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => '⚠️ Directional only',
|
||||
'value' => 'Pre/post comparison, not a randomised test. Cohorts are compared at equal age. Surge weeks (⚡, e.g. launch/YouTube) differ in acquisition channel and are not controlled — compare organic weeks like-for-like. Confidence builds over a quarter, not a single month.',
|
||||
'inline' => false,
|
||||
],
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
private function rateCell(?float $rate, bool $mature, int $eligible): string
|
||||
{
|
||||
if ($eligible === 0) {
|
||||
return '—';
|
||||
}
|
||||
|
||||
if (! $mature || $rate === null) {
|
||||
return 'pend';
|
||||
}
|
||||
|
||||
return ((int) round($rate * 100)).'%';
|
||||
}
|
||||
|
||||
private function aiCell(?float $rate, int $eligible): string
|
||||
{
|
||||
if ($eligible === 0 || $rate === null) {
|
||||
return '—';
|
||||
}
|
||||
|
||||
return ((int) round($rate * 100)).'%';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,213 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Ai;
|
||||
|
||||
use App\Models\AiConsent;
|
||||
use App\Models\User;
|
||||
use Carbon\CarbonImmutable;
|
||||
|
||||
class AiCohortReportCollector
|
||||
{
|
||||
private const ELIGIBILITY_WINDOW_DAYS = 7;
|
||||
|
||||
private const RETENTION_DAYS = 14;
|
||||
|
||||
private const TRIAL_DAYS = 14;
|
||||
|
||||
private const PAID_DAYS = 30;
|
||||
|
||||
private const DEFAULT_WEEKS = 16;
|
||||
|
||||
private const SURGE_MULTIPLIER = 2.5;
|
||||
|
||||
/**
|
||||
* Build the weekly eligible-cohort time series for the AI suggestions feature.
|
||||
*
|
||||
* Eligibility, metric horizons and maturity windows are all measured from
|
||||
* each user's own signup, so every weekly cohort is compared at the same
|
||||
* age regardless of the calendar.
|
||||
*
|
||||
* @return array{
|
||||
* releaseAt: ?CarbonImmutable,
|
||||
* releaseWeek: ?string,
|
||||
* weeks: list<array{
|
||||
* week: string,
|
||||
* weekStart: CarbonImmutable,
|
||||
* eligible: int,
|
||||
* retained: int,
|
||||
* retainedRate: ?float,
|
||||
* trial: int,
|
||||
* trialRate: ?float,
|
||||
* paid: int,
|
||||
* paidRate: ?float,
|
||||
* aiAccepted: int,
|
||||
* aiAcceptedRate: ?float,
|
||||
* phase: string,
|
||||
* surge: bool,
|
||||
* retentionMature: bool,
|
||||
* paidMature: bool,
|
||||
* }>
|
||||
* }
|
||||
*/
|
||||
public function collect(?int $weeks = null): array
|
||||
{
|
||||
$weeks = max(1, $weeks ?? (int) config('ai_suggestions.report.weeks', self::DEFAULT_WEEKS));
|
||||
|
||||
$now = CarbonImmutable::now('UTC');
|
||||
$windowStart = $now->startOfWeek(CarbonImmutable::MONDAY)->subWeeks($weeks - 1);
|
||||
|
||||
$releaseValue = AiConsent::query()->min('accepted_at');
|
||||
$releaseAt = $releaseValue !== null ? CarbonImmutable::parse($releaseValue) : null;
|
||||
$releaseWeekStart = $releaseAt?->startOfWeek(CarbonImmutable::MONDAY);
|
||||
|
||||
$aggregates = $this->aggregateEligibleUsers($windowStart);
|
||||
|
||||
$rows = [];
|
||||
$eligibleCounts = [];
|
||||
|
||||
for ($i = 0; $i < $weeks; $i++) {
|
||||
$weekStart = $windowStart->addWeeks($i);
|
||||
$weekEnd = $weekStart->endOfWeek(CarbonImmutable::SUNDAY);
|
||||
$key = (int) $weekStart->format('oW');
|
||||
|
||||
$agg = $aggregates[$key] ?? [
|
||||
'eligible' => 0,
|
||||
'retained' => 0,
|
||||
'trial' => 0,
|
||||
'paid' => 0,
|
||||
'aiAccepted' => 0,
|
||||
];
|
||||
|
||||
$eligible = $agg['eligible'];
|
||||
$retentionMature = $weekEnd->addDays(self::RETENTION_DAYS)->lessThanOrEqualTo($now);
|
||||
$paidMature = $weekEnd->addDays(self::PAID_DAYS)->lessThanOrEqualTo($now);
|
||||
|
||||
$eligibleCounts[] = $eligible;
|
||||
|
||||
$rows[$i] = [
|
||||
'week' => $weekStart->format('o-\WW'),
|
||||
'weekStart' => $weekStart,
|
||||
'eligible' => $eligible,
|
||||
'retained' => $agg['retained'],
|
||||
'retainedRate' => $retentionMature && $eligible > 0 ? $agg['retained'] / $eligible : null,
|
||||
'trial' => $agg['trial'],
|
||||
'trialRate' => $retentionMature && $eligible > 0 ? $agg['trial'] / $eligible : null,
|
||||
'paid' => $agg['paid'],
|
||||
'paidRate' => $paidMature && $eligible > 0 ? $agg['paid'] / $eligible : null,
|
||||
'aiAccepted' => $agg['aiAccepted'],
|
||||
'aiAcceptedRate' => $eligible > 0 ? $agg['aiAccepted'] / $eligible : null,
|
||||
'phase' => $this->phase($weekStart, $releaseWeekStart),
|
||||
'surge' => false,
|
||||
'retentionMature' => $retentionMature,
|
||||
'paidMature' => $paidMature,
|
||||
];
|
||||
}
|
||||
|
||||
$this->flagSurges($rows, $eligibleCounts);
|
||||
|
||||
return [
|
||||
'releaseAt' => $releaseAt,
|
||||
'releaseWeek' => $releaseWeekStart?->format('o-\WW'),
|
||||
'weeks' => array_values($rows),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate per-user metric flags for eligible users, keyed by ISO year-week.
|
||||
*
|
||||
* @return array<int, array{eligible: int, retained: int, trial: int, paid: int, aiAccepted: int}>
|
||||
*/
|
||||
private function aggregateEligibleUsers(CarbonImmutable $windowStart): array
|
||||
{
|
||||
$threshold = (int) config('ai_suggestions.eligibility_min_transactions', 50);
|
||||
$excluded = (array) config('ai_suggestions.report.excluded_emails', []);
|
||||
|
||||
$rows = User::query()
|
||||
->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded))
|
||||
->where('users.created_at', '>=', $windowStart)
|
||||
->whereHas('transactions', function ($query): void {
|
||||
$query->whereRaw(
|
||||
'transactions.created_at <= DATE_ADD(users.created_at, INTERVAL '.self::ELIGIBILITY_WINDOW_DAYS.' DAY)',
|
||||
);
|
||||
}, '>=', $threshold)
|
||||
->selectRaw('YEARWEEK(users.created_at, 3) as yearweek')
|
||||
->selectRaw('(users.last_active_at IS NOT NULL AND users.last_active_at >= DATE_ADD(users.created_at, INTERVAL '.self::RETENTION_DAYS.' DAY)) as retained')
|
||||
->selectRaw('EXISTS(SELECT 1 FROM subscriptions s WHERE s.user_id = users.id AND s.created_at <= DATE_ADD(users.created_at, INTERVAL '.self::TRIAL_DAYS.' DAY)) as has_trial')
|
||||
->selectRaw("EXISTS(SELECT 1 FROM subscriptions s WHERE s.user_id = users.id AND s.stripe_status = 'active' AND s.created_at <= DATE_ADD(users.created_at, INTERVAL ".self::PAID_DAYS.' DAY)) as has_paid')
|
||||
->selectRaw('EXISTS(SELECT 1 FROM ai_consents c WHERE c.user_id = users.id AND c.accepted_at IS NOT NULL) as ai_accepted')
|
||||
->toBase()
|
||||
->get();
|
||||
|
||||
$aggregates = [];
|
||||
|
||||
foreach ($rows as $row) {
|
||||
$key = (int) $row->yearweek;
|
||||
|
||||
if (! isset($aggregates[$key])) {
|
||||
$aggregates[$key] = ['eligible' => 0, 'retained' => 0, 'trial' => 0, 'paid' => 0, 'aiAccepted' => 0];
|
||||
}
|
||||
|
||||
$aggregates[$key]['eligible']++;
|
||||
$aggregates[$key]['retained'] += (int) $row->retained;
|
||||
$aggregates[$key]['trial'] += (int) $row->has_trial;
|
||||
$aggregates[$key]['paid'] += (int) $row->has_paid;
|
||||
$aggregates[$key]['aiAccepted'] += (int) $row->ai_accepted;
|
||||
}
|
||||
|
||||
return $aggregates;
|
||||
}
|
||||
|
||||
private function phase(CarbonImmutable $weekStart, ?CarbonImmutable $releaseWeekStart): string
|
||||
{
|
||||
if ($releaseWeekStart === null) {
|
||||
return 'pre';
|
||||
}
|
||||
|
||||
return $weekStart->lessThan($releaseWeekStart) ? 'pre' : 'post';
|
||||
}
|
||||
|
||||
/**
|
||||
* Flag weeks whose eligible volume is an outlier (e.g. a launch/marketing
|
||||
* spike) so a non-representative acquisition wave can't be read as an
|
||||
* organic trend.
|
||||
*
|
||||
* @param array<int, array<string, mixed>> $rows
|
||||
* @param list<int> $eligibleCounts
|
||||
*/
|
||||
private function flagSurges(array &$rows, array $eligibleCounts): void
|
||||
{
|
||||
$nonZero = array_values(array_filter($eligibleCounts, fn (int $count): bool => $count > 0));
|
||||
$median = $this->median($nonZero);
|
||||
|
||||
if ($median <= 0.0) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($rows as $index => $row) {
|
||||
if ($row['eligible'] > self::SURGE_MULTIPLIER * $median) {
|
||||
$rows[$index]['surge'] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<int> $values
|
||||
*/
|
||||
private function median(array $values): float
|
||||
{
|
||||
sort($values);
|
||||
$count = count($values);
|
||||
|
||||
if ($count === 0) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
$middle = intdiv($count, 2);
|
||||
|
||||
if ($count % 2 === 1) {
|
||||
return (float) $values[$middle];
|
||||
}
|
||||
|
||||
return ($values[$middle - 1] + $values[$middle]) / 2;
|
||||
}
|
||||
}
|
||||
|
|
@ -108,4 +108,26 @@ return [
|
|||
|
||||
'consent_version' => (string) env('AI_SUGGESTIONS_CONSENT_VERSION', '1'),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Cohort Report
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Settings for the weekly AI-suggestions cohort report (the
|
||||
| `stats:ai-cohort-report` command). `weeks` is how many weekly cohorts to
|
||||
| include; `excluded_emails` is a comma-separated list of staff/test
|
||||
| accounts (e.g. the user who plants the release-anchor consent) that must
|
||||
| be kept out of the cohort metrics.
|
||||
|
|
||||
*/
|
||||
|
||||
'report' => [
|
||||
'weeks' => (int) env('AI_SUGGESTIONS_REPORT_WEEKS', 16),
|
||||
|
||||
'excluded_emails' => array_values(array_filter(array_map(
|
||||
'trim',
|
||||
explode(',', (string) env('AI_SUGGESTIONS_REPORT_EXCLUDED_EMAILS', '')),
|
||||
))),
|
||||
],
|
||||
|
||||
];
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ return [
|
|||
|
||||
'discord' => [
|
||||
'webhook_url' => env('DISCORD_WEBHOOK_URL'),
|
||||
'ai_cohort_webhook_url' => env('DISCORD_AI_COHORT_WEBHOOK_URL'),
|
||||
],
|
||||
|
||||
];
|
||||
|
|
|
|||
|
|
@ -12,3 +12,4 @@ Schedule::command('resend:sync-leads')->dailyAt('03:00');
|
|||
Schedule::command('leads:send-invitations --force --limit=150')->dailyAt('09:00');
|
||||
Schedule::command('leads:send-re-invitations --force')->dailyAt('09:00');
|
||||
Schedule::command('stats:daily-report')->dailyAt('09:00')->timezone('Europe/Madrid');
|
||||
Schedule::command('stats:ai-cohort-report')->monthlyOn(1, '09:00')->timezone('Europe/Madrid');
|
||||
|
|
|
|||
|
|
@ -0,0 +1,224 @@
|
|||
<?php
|
||||
|
||||
use App\Models\Account;
|
||||
use App\Models\AiConsent;
|
||||
use App\Models\Category;
|
||||
use App\Models\Transaction;
|
||||
use App\Models\User;
|
||||
use App\Services\Ai\AiCohortReportCollector;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
use function Pest\Laravel\artisan;
|
||||
|
||||
function referenceNow(): CarbonImmutable
|
||||
{
|
||||
return CarbonImmutable::create(2026, 6, 17, 12, 0, 0, 'UTC');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an eligible (or deliberately ineligible) user with transactions, and
|
||||
* optionally a consent and a subscription, all anchored to the given signup.
|
||||
*
|
||||
* @param array<string, mixed> $opts
|
||||
*/
|
||||
function cohortUser(CarbonImmutable $signup, array $opts = []): User
|
||||
{
|
||||
$user = User::factory()->create([
|
||||
'email' => $opts['email'] ?? fake()->unique()->safeEmail(),
|
||||
'created_at' => $signup,
|
||||
'last_active_at' => $opts['lastActiveAt'] ?? null,
|
||||
]);
|
||||
|
||||
$account = Account::factory()->for($user)->create();
|
||||
$category = Category::factory()->for($user)->create();
|
||||
|
||||
Transaction::factory()->count($opts['transactions'] ?? 3)->for($user)->for($account)->create([
|
||||
'category_id' => $category->id,
|
||||
'created_at' => $signup->addDay(),
|
||||
]);
|
||||
|
||||
if (! empty($opts['consentAt'])) {
|
||||
AiConsent::factory()->for($user)->create(['accepted_at' => $opts['consentAt']]);
|
||||
}
|
||||
|
||||
if (! empty($opts['subscription'])) {
|
||||
$user->subscriptions()->create([
|
||||
'type' => 'default',
|
||||
'stripe_id' => 'sub_'.Str::random(12),
|
||||
'stripe_status' => $opts['subscription']['status'],
|
||||
'stripe_price' => 'price_test',
|
||||
'created_at' => $opts['subscription']['at'],
|
||||
]);
|
||||
}
|
||||
|
||||
return $user;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{weeks: list<array<string, mixed>>} $report
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
function rowForWeek(array $report, CarbonImmutable $signup): array
|
||||
{
|
||||
$label = $signup->startOfWeek(CarbonImmutable::MONDAY)->format('o-\WW');
|
||||
|
||||
foreach ($report['weeks'] as $row) {
|
||||
if ($row['week'] === $label) {
|
||||
return $row;
|
||||
}
|
||||
}
|
||||
|
||||
throw new RuntimeException("No cohort row found for week {$label}");
|
||||
}
|
||||
|
||||
beforeEach(function () {
|
||||
Carbon::setTestNow(referenceNow());
|
||||
config(['ai_suggestions.eligibility_min_transactions' => 3]);
|
||||
config(['ai_suggestions.report.excluded_emails' => []]);
|
||||
});
|
||||
|
||||
it('only counts users who imported enough transactions within their first week', function () {
|
||||
$signup = referenceNow()->subWeeks(6);
|
||||
|
||||
cohortUser($signup, ['transactions' => 3, 'lastActiveAt' => $signup->addDays(20)]); // eligible, retained
|
||||
cohortUser($signup, ['transactions' => 3, 'lastActiveAt' => $signup->addDays(2)]); // eligible, churned early
|
||||
cohortUser($signup, ['transactions' => 2, 'lastActiveAt' => $signup->addDays(20)]); // ineligible (<3 txns)
|
||||
|
||||
// 3 transactions, but imported on day 10 — outside the 7-day eligibility window.
|
||||
$late = cohortUser($signup, ['transactions' => 0, 'lastActiveAt' => $signup->addDays(20)]);
|
||||
$account = Account::factory()->for($late)->create();
|
||||
Transaction::factory()->count(3)->for($late)->for($account)->create(['created_at' => $signup->addDays(10)]);
|
||||
|
||||
$report = app(AiCohortReportCollector::class)->collect();
|
||||
$row = rowForWeek($report, $signup);
|
||||
|
||||
expect($row['eligible'])->toBe(2)
|
||||
->and($row['retained'])->toBe(1)
|
||||
->and($row['retainedRate'])->toBe(0.5)
|
||||
->and($row['retentionMature'])->toBeTrue();
|
||||
});
|
||||
|
||||
it('counts trial starts within 14 days and active paid conversions within 30 days', function () {
|
||||
$signup = referenceNow()->subWeeks(6);
|
||||
|
||||
// Trialing subscription started at signup: counts as trial, not as paid.
|
||||
cohortUser($signup, [
|
||||
'transactions' => 3,
|
||||
'subscription' => ['status' => 'trialing', 'at' => $signup->addDays(2)],
|
||||
]);
|
||||
|
||||
// Active subscription within 30 days: counts as both trial and paid.
|
||||
cohortUser($signup, [
|
||||
'transactions' => 3,
|
||||
'subscription' => ['status' => 'active', 'at' => $signup->addDays(5)],
|
||||
]);
|
||||
|
||||
// Active subscription, but only after day 40: outside both windows.
|
||||
cohortUser($signup, [
|
||||
'transactions' => 3,
|
||||
'subscription' => ['status' => 'active', 'at' => $signup->addDays(40)],
|
||||
]);
|
||||
|
||||
$report = app(AiCohortReportCollector::class)->collect();
|
||||
$row = rowForWeek($report, $signup);
|
||||
|
||||
expect($row['eligible'])->toBe(3)
|
||||
->and($row['trial'])->toBe(2)
|
||||
->and($row['paid'])->toBe(1)
|
||||
->and($row['paidMature'])->toBeTrue();
|
||||
});
|
||||
|
||||
it('derives the release anchor from the first consent and splits cohorts pre/post', function () {
|
||||
config(['ai_suggestions.report.excluded_emails' => ['staff@whisper.test']]);
|
||||
|
||||
$release = referenceNow()->subWeeks(4);
|
||||
|
||||
// Staff account plants the release-anchor consent; excluded from cohort metrics.
|
||||
cohortUser($release, [
|
||||
'email' => 'staff@whisper.test',
|
||||
'transactions' => 3,
|
||||
'consentAt' => $release,
|
||||
]);
|
||||
|
||||
$before = referenceNow()->subWeeks(6);
|
||||
$after = referenceNow()->subWeeks(2);
|
||||
cohortUser($before, ['transactions' => 3]);
|
||||
cohortUser($after, ['transactions' => 3]);
|
||||
|
||||
$report = app(AiCohortReportCollector::class)->collect();
|
||||
|
||||
expect($report['releaseWeek'])->toBe($release->startOfWeek(CarbonImmutable::MONDAY)->format('o-\WW'))
|
||||
->and(rowForWeek($report, $before)['phase'])->toBe('pre')
|
||||
->and(rowForWeek($report, $after)['phase'])->toBe('post')
|
||||
// Staff account was excluded, so the release week has no eligible users.
|
||||
->and(rowForWeek($report, $release)['eligible'])->toBe(0);
|
||||
});
|
||||
|
||||
it('marks cohorts as not mature until the metric horizon has elapsed', function () {
|
||||
$recent = referenceNow()->subDays(3); // current week — too young for any 14d metric
|
||||
$midAged = referenceNow()->subWeeks(3); // older than 14d, younger than 30d
|
||||
|
||||
cohortUser($recent, ['transactions' => 3, 'lastActiveAt' => $recent->addDay()]);
|
||||
cohortUser($midAged, ['transactions' => 3, 'lastActiveAt' => $midAged->addDays(20)]);
|
||||
|
||||
$report = app(AiCohortReportCollector::class)->collect();
|
||||
|
||||
$recentRow = rowForWeek($report, $recent);
|
||||
expect($recentRow['retentionMature'])->toBeFalse()
|
||||
->and($recentRow['retainedRate'])->toBeNull()
|
||||
->and($recentRow['paidRate'])->toBeNull();
|
||||
|
||||
$midRow = rowForWeek($report, $midAged);
|
||||
expect($midRow['retentionMature'])->toBeTrue()
|
||||
->and($midRow['retainedRate'])->not->toBeNull()
|
||||
->and($midRow['paidMature'])->toBeFalse()
|
||||
->and($midRow['paidRate'])->toBeNull();
|
||||
});
|
||||
|
||||
it('flags weeks whose eligible volume is an outlier surge', function () {
|
||||
$surgeWeek = referenceNow()->subWeeks(5);
|
||||
$quietWeeks = [referenceNow()->subWeeks(8), referenceNow()->subWeeks(7), referenceNow()->subWeeks(6)];
|
||||
|
||||
foreach ($quietWeeks as $week) {
|
||||
cohortUser($week, ['transactions' => 3]);
|
||||
}
|
||||
|
||||
for ($i = 0; $i < 6; $i++) {
|
||||
cohortUser($surgeWeek, ['transactions' => 3]);
|
||||
}
|
||||
|
||||
$report = app(AiCohortReportCollector::class)->collect();
|
||||
|
||||
expect(rowForWeek($report, $surgeWeek)['surge'])->toBeTrue()
|
||||
->and(rowForWeek($report, $quietWeeks[0])['surge'])->toBeFalse();
|
||||
});
|
||||
|
||||
it('posts the cohort report embed to the configured discord webhook', function () {
|
||||
config(['services.discord.ai_cohort_webhook_url' => 'https://discord.test/hook']);
|
||||
Http::fake(['discord.test/*' => Http::response('', 204)]);
|
||||
|
||||
cohortUser(referenceNow()->subWeeks(6), ['transactions' => 3, 'lastActiveAt' => referenceNow()->subWeeks(3)]);
|
||||
|
||||
artisan('stats:ai-cohort-report')->assertSuccessful();
|
||||
|
||||
Http::assertSent(function ($request) {
|
||||
return $request->url() === 'https://discord.test/hook'
|
||||
&& isset($request['embeds'][0]['title'])
|
||||
&& str_contains($request['embeds'][0]['title'], 'AI Suggestions');
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to the default discord webhook when no dedicated one is set', function () {
|
||||
config([
|
||||
'services.discord.ai_cohort_webhook_url' => null,
|
||||
'services.discord.webhook_url' => 'https://discord.test/default',
|
||||
]);
|
||||
Http::fake(['discord.test/*' => Http::response('', 204)]);
|
||||
|
||||
artisan('stats:ai-cohort-report')->assertSuccessful();
|
||||
|
||||
Http::assertSent(fn ($request) => $request->url() === 'https://discord.test/default');
|
||||
});
|
||||
Loading…
Reference in New Issue