perf(ai): memoize the description corpus per user in AiRuleLearner

learnFromCorrection() reloaded and re-tokenized every one of the user's
descriptions on each correction. In a bulk category update that runs once
per transaction (part of the N+1 in PHP-LARAVEL-40). The corpus is
immutable while only categories change, so memoize the document-frequency
map and corpus size per user for the instance's lifetime.
This commit is contained in:
Víctor Falcón 2026-07-03 03:37:39 +02:00
parent 7b13214e07
commit e5780eade7
2 changed files with 74 additions and 8 deletions

View File

@ -27,6 +27,17 @@ use Illuminate\Support\Str;
*/
class AiRuleLearner
{
/**
* Per-user document-frequency corpus, memoized for the lifetime of this
* instance. A bulk correction runs learnFromCorrection once per transaction
* for the same user, and the description corpus is immutable while only
* categories change so loading and tokenizing every description on every
* transaction (the N+1 in PHP-LARAVEL-40) is wasted work.
*
* @var array<string, array{frequency: array<string, int>, count: int}>
*/
private array $descriptionCorpus = [];
public function __construct(
private readonly DescriptionTokenizer $tokenizer,
private readonly TransactionMatcher $matcher,
@ -153,16 +164,33 @@ class AiRuleLearner
*/
private function distinctiveDescriptionTokens(Transaction $transaction): array
{
$descriptions = Transaction::query()
->where('user_id', $transaction->user_id)
->whereNull('description_iv')
->pluck('description')
->all();
$corpus = $this->descriptionCorpus($transaction->user_id);
$threshold = $corpus['count'] * (float) config('ai_suggestions.noise_token_fraction');
$frequency = $this->tokenizer->documentFrequency($descriptions);
$threshold = count($descriptions) * (float) config('ai_suggestions.noise_token_fraction');
return $this->tokenizer->distinctiveTokens((string) $transaction->description, $corpus['frequency'], $threshold);
}
return $this->tokenizer->distinctiveTokens((string) $transaction->description, $frequency, $threshold);
/**
* The user's description document-frequency map and corpus size, loaded once
* per instance. Safe to memoize: descriptions are never mutated by a
* categorization change, so the corpus is stable across a bulk correction.
*
* @return array{frequency: array<string, int>, count: int}
*/
private function descriptionCorpus(string $userId): array
{
return $this->descriptionCorpus[$userId] ??= (function () use ($userId): array {
$descriptions = Transaction::query()
->where('user_id', $userId)
->whereNull('description_iv')
->pluck('description')
->all();
return [
'frequency' => $this->tokenizer->documentFrequency($descriptions),
'count' => count($descriptions),
];
})();
}
/**

View File

@ -10,6 +10,7 @@ use App\Models\User;
use App\Services\Ai\AiRuleLearner;
use App\Services\Ai\CategorizationOutcome;
use App\Services\AutomationRuleService;
use Illuminate\Support\Facades\DB;
function expenseCategory(User $user): Category
{
@ -51,6 +52,43 @@ it('creates an ai-owned rule at the lowest priority and links the transaction',
->and($transaction->refresh()->categorized_by_rule_id)->toBe($rule->id);
});
it('loads the description corpus once across a bulk of correction learnings', function () {
$user = User::factory()->create();
$category = expenseCategory($user);
// Merchant-less, plaintext transactions force the description-token path,
// which is what loads the per-user description corpus.
$makeTxn = fn (string $description): Transaction => Transaction::factory()->plaintext()->create([
'user_id' => $user->id,
'category_id' => null,
'creditor_name' => null,
'debtor_name' => null,
'description' => $description,
]);
$first = $makeTxn('zzalpha distinctive ref one');
$second = $makeTxn('zzbeta distinctive ref two');
// Same instance across the batch, mirroring the bulkUpdate loop where the
// handler (and its learner) is resolved once.
$learner = app(AiRuleLearner::class);
DB::enableQueryLog();
$learner->learnFromCorrection($first, $category->id);
$learner->learnFromCorrection($second, $category->id);
$queries = collect(DB::getQueryLog());
DB::disableQueryLog();
// The corpus is the pluck of the `description` column (not the matcher's
// count(*) probes, which also filter on description_iv).
$corpusLoads = $queries->filter(fn (array $q): bool => str_starts_with(strtolower(ltrim($q['query'])), 'select')
&& str_contains($q['query'], 'description_iv')
&& ! str_contains(strtolower($q['query']), 'count(')
);
expect($corpusLoads)->toHaveCount(1);
});
it('appends a new merchant to the existing ai rule for the same category', function () {
$user = User::factory()->create();
$category = expenseCategory($user);