feat(stats): surface revenue-map gaps in the price experiment report
The collector now returns the price ids that carried net-active subscriptions but were missing from the monthly-equivalent map, and the report prints a loud warning (Stripe unreachable, or unmapped prices undercounting MRR as 0) where the decision-maker reads it — previously this only went to the logs, so an unsynced arm price with no current payers could bias CM unnoticed.
This commit is contained in:
parent
5686617cf8
commit
f082e691fb
|
|
@ -46,7 +46,7 @@ class SendPriceExperimentFunnelReportCommand extends Command
|
|||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
foreach ([...$this->tableLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)] as $line) {
|
||||
foreach ([...$this->tableLines($report), ...$this->integrityLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)] as $line) {
|
||||
$this->line($line);
|
||||
}
|
||||
|
||||
|
|
@ -103,6 +103,35 @@ class SendPriceExperimentFunnelReportCommand extends Command
|
|||
return $lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Revenue-map integrity warnings, surfaced in the report the decision-maker
|
||||
* reads (not just the logs): Stripe unreachable, or net-active subscriptions on
|
||||
* price ids missing from the monthly-equivalent map — which would silently
|
||||
* undercount MRR/CM as 0 and bias the comparison against whichever arm's price
|
||||
* is unmapped. Empty (no lines) when everything resolves.
|
||||
*
|
||||
* @param array{revenueAvailable: bool, unmappedPriceIds: list<string>} $report
|
||||
* @return list<string>
|
||||
*/
|
||||
private function integrityLines(array $report): array
|
||||
{
|
||||
$lines = [];
|
||||
|
||||
if (! $report['revenueAvailable']) {
|
||||
$lines[] = '';
|
||||
$lines[] = '⚠ REVENUE UNAVAILABLE — Stripe prices could not be loaded; MRR/CM are blank this run.';
|
||||
}
|
||||
|
||||
if (($report['unmappedPriceIds'] ?? []) !== []) {
|
||||
$lines[] = '';
|
||||
$lines[] = '⚠ UNMAPPED PRICES — net-active subs on price ids absent from the revenue map '
|
||||
.'(MRR undercounted as 0): '.implode(', ', $report['unmappedPriceIds'])
|
||||
.'. Confirm both arm prices are synced under the pro product (stripe:sync-prices).';
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample-ratio-mismatch check: the split must be ~50/50. A low p means the
|
||||
* assignment (or the pipeline that filters it) is broken, which invalidates
|
||||
|
|
@ -238,14 +267,14 @@ class SendPriceExperimentFunnelReportCommand extends Command
|
|||
}
|
||||
|
||||
/**
|
||||
* @param array{startedAt: CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, decisionWindowDays: int, variants: array<string, array<string, mixed>>} $report
|
||||
* @param array{startedAt: CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, decisionWindowDays: int, unmappedPriceIds: list<string>, variants: array<string, array<string, mixed>>} $report
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function buildEmbed(array $report): array
|
||||
{
|
||||
return [
|
||||
'title' => '💶 Price Experiment — Funnel (control vs high)',
|
||||
'description' => "```\n".implode("\n", [...$this->tableLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)])."\n```",
|
||||
'description' => "```\n".implode("\n", [...$this->tableLines($report), ...$this->integrityLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)])."\n```",
|
||||
'color' => 0x57F287,
|
||||
'fields' => [
|
||||
[
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ class PriceExperimentFunnelCollector
|
|||
* revenueAvailable: bool,
|
||||
* costPerConnectionCents: int,
|
||||
* decisionWindowDays: int,
|
||||
* unmappedPriceIds: list<string>,
|
||||
* variants: array<string, array{
|
||||
* assigned: int, activated: int, subscribed: int,
|
||||
* assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int,
|
||||
|
|
@ -76,6 +77,7 @@ class PriceExperimentFunnelCollector
|
|||
'revenueAvailable' => false,
|
||||
'costPerConnectionCents' => $costPerConnectionCents,
|
||||
'decisionWindowDays' => $window,
|
||||
'unmappedPriceIds' => [],
|
||||
'variants' => $variants,
|
||||
];
|
||||
}
|
||||
|
|
@ -209,6 +211,7 @@ class PriceExperimentFunnelCollector
|
|||
'revenueAvailable' => $monthlyEquiv !== [],
|
||||
'costPerConnectionCents' => $costPerConnectionCents,
|
||||
'decisionWindowDays' => $window,
|
||||
'unmappedPriceIds' => array_keys($missingPrices),
|
||||
'variants' => $variants,
|
||||
];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use App\Models\User;
|
|||
use App\Services\Stats\PriceExperimentFunnelCollector;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Artisan;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Str;
|
||||
|
|
@ -175,6 +176,19 @@ it('prints the primary CM/user (Welch) and conversion guardrail blocks', functio
|
|||
->assertSuccessful();
|
||||
});
|
||||
|
||||
it('surfaces an unmapped-price warning in the report, not just the logs', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// Net-active sub on a price id the seeded map does not know → MRR silently 0.
|
||||
priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_rotated_old']);
|
||||
|
||||
Artisan::call('stats:price-experiment-funnel', ['--no-discord' => true]);
|
||||
$output = Artisan::output();
|
||||
|
||||
expect($output)->toContain('UNMAPPED PRICES')
|
||||
->and($output)->toContain('price_rotated_old');
|
||||
});
|
||||
|
||||
it('flags a sample-ratio mismatch when the assignment split is lopsided', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue