perf(ai): memoize the description corpus per user in AiRuleLearner
learnFromCorrection() reloaded and re-tokenized every one of the user's descriptions on each correction. In a bulk category update that runs once per transaction (part of the N+1 in PHP-LARAVEL-40). The corpus is immutable while only categories change, so memoize the document-frequency map and corpus size per user for the instance's lifetime.
This commit is contained in:
parent
7b13214e07
commit
e5780eade7
|
|
@ -27,6 +27,17 @@ use Illuminate\Support\Str;
|
|||
*/
|
||||
class AiRuleLearner
|
||||
{
|
||||
/**
|
||||
* Per-user document-frequency corpus, memoized for the lifetime of this
|
||||
* instance. A bulk correction runs learnFromCorrection once per transaction
|
||||
* for the same user, and the description corpus is immutable while only
|
||||
* categories change — so loading and tokenizing every description on every
|
||||
* transaction (the N+1 in PHP-LARAVEL-40) is wasted work.
|
||||
*
|
||||
* @var array<string, array{frequency: array<string, int>, count: int}>
|
||||
*/
|
||||
private array $descriptionCorpus = [];
|
||||
|
||||
public function __construct(
|
||||
private readonly DescriptionTokenizer $tokenizer,
|
||||
private readonly TransactionMatcher $matcher,
|
||||
|
|
@ -153,16 +164,33 @@ class AiRuleLearner
|
|||
*/
|
||||
private function distinctiveDescriptionTokens(Transaction $transaction): array
|
||||
{
|
||||
$descriptions = Transaction::query()
|
||||
->where('user_id', $transaction->user_id)
|
||||
->whereNull('description_iv')
|
||||
->pluck('description')
|
||||
->all();
|
||||
$corpus = $this->descriptionCorpus($transaction->user_id);
|
||||
$threshold = $corpus['count'] * (float) config('ai_suggestions.noise_token_fraction');
|
||||
|
||||
$frequency = $this->tokenizer->documentFrequency($descriptions);
|
||||
$threshold = count($descriptions) * (float) config('ai_suggestions.noise_token_fraction');
|
||||
return $this->tokenizer->distinctiveTokens((string) $transaction->description, $corpus['frequency'], $threshold);
|
||||
}
|
||||
|
||||
return $this->tokenizer->distinctiveTokens((string) $transaction->description, $frequency, $threshold);
|
||||
/**
|
||||
* The user's description document-frequency map and corpus size, loaded once
|
||||
* per instance. Safe to memoize: descriptions are never mutated by a
|
||||
* categorization change, so the corpus is stable across a bulk correction.
|
||||
*
|
||||
* @return array{frequency: array<string, int>, count: int}
|
||||
*/
|
||||
private function descriptionCorpus(string $userId): array
|
||||
{
|
||||
return $this->descriptionCorpus[$userId] ??= (function () use ($userId): array {
|
||||
$descriptions = Transaction::query()
|
||||
->where('user_id', $userId)
|
||||
->whereNull('description_iv')
|
||||
->pluck('description')
|
||||
->all();
|
||||
|
||||
return [
|
||||
'frequency' => $this->tokenizer->documentFrequency($descriptions),
|
||||
'count' => count($descriptions),
|
||||
];
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ use App\Models\User;
|
|||
use App\Services\Ai\AiRuleLearner;
|
||||
use App\Services\Ai\CategorizationOutcome;
|
||||
use App\Services\AutomationRuleService;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
function expenseCategory(User $user): Category
|
||||
{
|
||||
|
|
@ -51,6 +52,43 @@ it('creates an ai-owned rule at the lowest priority and links the transaction',
|
|||
->and($transaction->refresh()->categorized_by_rule_id)->toBe($rule->id);
|
||||
});
|
||||
|
||||
it('loads the description corpus once across a bulk of correction learnings', function () {
|
||||
$user = User::factory()->create();
|
||||
$category = expenseCategory($user);
|
||||
|
||||
// Merchant-less, plaintext transactions force the description-token path,
|
||||
// which is what loads the per-user description corpus.
|
||||
$makeTxn = fn (string $description): Transaction => Transaction::factory()->plaintext()->create([
|
||||
'user_id' => $user->id,
|
||||
'category_id' => null,
|
||||
'creditor_name' => null,
|
||||
'debtor_name' => null,
|
||||
'description' => $description,
|
||||
]);
|
||||
|
||||
$first = $makeTxn('zzalpha distinctive ref one');
|
||||
$second = $makeTxn('zzbeta distinctive ref two');
|
||||
|
||||
// Same instance across the batch, mirroring the bulkUpdate loop where the
|
||||
// handler (and its learner) is resolved once.
|
||||
$learner = app(AiRuleLearner::class);
|
||||
|
||||
DB::enableQueryLog();
|
||||
$learner->learnFromCorrection($first, $category->id);
|
||||
$learner->learnFromCorrection($second, $category->id);
|
||||
$queries = collect(DB::getQueryLog());
|
||||
DB::disableQueryLog();
|
||||
|
||||
// The corpus is the pluck of the `description` column (not the matcher's
|
||||
// count(*) probes, which also filter on description_iv).
|
||||
$corpusLoads = $queries->filter(fn (array $q): bool => str_starts_with(strtolower(ltrim($q['query'])), 'select')
|
||||
&& str_contains($q['query'], 'description_iv')
|
||||
&& ! str_contains(strtolower($q['query']), 'count(')
|
||||
);
|
||||
|
||||
expect($corpusLoads)->toHaveCount(1);
|
||||
});
|
||||
|
||||
it('appends a new merchant to the existing ai rule for the same category', function () {
|
||||
$user = User::factory()->create();
|
||||
$category = expenseCategory($user);
|
||||
|
|
|
|||
Loading…
Reference in New Issue