feat(stats): surface revenue-map gaps in the price experiment report

The collector now returns the price ids that carried net-active subscriptions but
were missing from the monthly-equivalent map, and the report prints a loud warning
(Stripe unreachable, or unmapped prices undercounting MRR as 0) where the
decision-maker reads it — previously this only went to the logs, so an unsynced
arm price with no current payers could bias CM unnoticed.
This commit is contained in:
Víctor Falcón 2026-07-18 19:23:32 +02:00
parent 5686617cf8
commit f082e691fb
3 changed files with 49 additions and 3 deletions

View File

@ -46,7 +46,7 @@ class SendPriceExperimentFunnelReportCommand extends Command
return self::SUCCESS;
}
foreach ([...$this->tableLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)] as $line) {
foreach ([...$this->tableLines($report), ...$this->integrityLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)] as $line) {
$this->line($line);
}
@ -103,6 +103,35 @@ class SendPriceExperimentFunnelReportCommand extends Command
return $lines;
}
/**
* Revenue-map integrity warnings, surfaced in the report the decision-maker
* reads (not just the logs): Stripe unreachable, or net-active subscriptions on
* price ids missing from the monthly-equivalent map which would silently
* undercount MRR/CM as 0 and bias the comparison against whichever arm's price
* is unmapped. Empty (no lines) when everything resolves.
*
* @param array{revenueAvailable: bool, unmappedPriceIds: list<string>} $report
* @return list<string>
*/
private function integrityLines(array $report): array
{
$lines = [];
if (! $report['revenueAvailable']) {
$lines[] = '';
$lines[] = '⚠ REVENUE UNAVAILABLE — Stripe prices could not be loaded; MRR/CM are blank this run.';
}
if (($report['unmappedPriceIds'] ?? []) !== []) {
$lines[] = '';
$lines[] = '⚠ UNMAPPED PRICES — net-active subs on price ids absent from the revenue map '
.'(MRR undercounted as 0): '.implode(', ', $report['unmappedPriceIds'])
.'. Confirm both arm prices are synced under the pro product (stripe:sync-prices).';
}
return $lines;
}
/**
* Sample-ratio-mismatch check: the split must be ~50/50. A low p means the
* assignment (or the pipeline that filters it) is broken, which invalidates
@ -238,14 +267,14 @@ class SendPriceExperimentFunnelReportCommand extends Command
}
/**
* @param array{startedAt: CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, decisionWindowDays: int, variants: array<string, array<string, mixed>>} $report
* @param array{startedAt: CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, decisionWindowDays: int, unmappedPriceIds: list<string>, variants: array<string, array<string, mixed>>} $report
* @return array<string, mixed>
*/
private function buildEmbed(array $report): array
{
return [
'title' => '💶 Price Experiment — Funnel (control vs high)',
'description' => "```\n".implode("\n", [...$this->tableLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)])."\n```",
'description' => "```\n".implode("\n", [...$this->tableLines($report), ...$this->integrityLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)])."\n```",
'color' => 0x57F287,
'fields' => [
[

View File

@ -47,6 +47,7 @@ class PriceExperimentFunnelCollector
* revenueAvailable: bool,
* costPerConnectionCents: int,
* decisionWindowDays: int,
* unmappedPriceIds: list<string>,
* variants: array<string, array{
* assigned: int, activated: int, subscribed: int,
* assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int,
@ -76,6 +77,7 @@ class PriceExperimentFunnelCollector
'revenueAvailable' => false,
'costPerConnectionCents' => $costPerConnectionCents,
'decisionWindowDays' => $window,
'unmappedPriceIds' => [],
'variants' => $variants,
];
}
@ -209,6 +211,7 @@ class PriceExperimentFunnelCollector
'revenueAvailable' => $monthlyEquiv !== [],
'costPerConnectionCents' => $costPerConnectionCents,
'decisionWindowDays' => $window,
'unmappedPriceIds' => array_keys($missingPrices),
'variants' => $variants,
];
}

View File

@ -7,6 +7,7 @@ use App\Models\User;
use App\Services\Stats\PriceExperimentFunnelCollector;
use Carbon\CarbonImmutable;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Str;
@ -175,6 +176,19 @@ it('prints the primary CM/user (Welch) and conversion guardrail blocks', functio
->assertSuccessful();
});
it('surfaces an unmapped-price warning in the report, not just the logs', function () {
$signup = CarbonImmutable::parse('2026-06-05');
// Net-active sub on a price id the seeded map does not know → MRR silently 0.
priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_rotated_old']);
Artisan::call('stats:price-experiment-funnel', ['--no-discord' => true]);
$output = Artisan::output();
expect($output)->toContain('UNMAPPED PRICES')
->and($output)->toContain('price_rotated_old');
});
it('flags a sample-ratio mismatch when the assignment split is lopsided', function () {
$signup = CarbonImmutable::parse('2026-06-05');