feat(ai): learn a forward rule from category corrections
When a user corrects a category the AI assigned, the system now records the correction as a deterministic, forward-looking automation rule so the same merchant — or the same distinctive description — is never mis-categorized the same way again. The next matching transaction is caught by the rule before the model ever runs, ending the correct-it-every-time loop. - New RuleOrigin::Correction: rules learned from corrections, protected from the AI self-heal and editable like any rule in settings. - AiRuleLearner::learnFromCorrection keys on the merchant when present (stable as the description varies), otherwise on the description's distinctive tokens, guarded against over-broad description rules that could silently mis-file en masse. - A key lives in exactly one correction rule, so changing your mind moves it to the new category. Correcting a transaction a prior correction rule categorized is also learnable, so correction rules stay fixable. - Extract DescriptionTokenizer, shared by RuleSuggestionAggregator and the learner, removing the duplicated distinctive-token logic. - TransactionMatcher::countMatchingAll backs the over-broad guard. - The update endpoint returns the learned rule; the transactions table shows a toast with an instant Undo.
This commit is contained in:
Víctor Falcón
parent
300756e553
commit
88b43dfb99
|
|
@ -6,12 +6,14 @@ enum RuleOrigin: string
|
|||
{
|
||||
case User = 'user';
|
||||
case Ai = 'ai';
|
||||
case Correction = 'correction';
|
||||
|
||||
public function label(): string
|
||||
{
|
||||
return match ($this) {
|
||||
self::User => 'User',
|
||||
self::Ai => 'AI',
|
||||
self::Correction => 'Correction',
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -19,4 +21,13 @@ enum RuleOrigin: string
|
|||
{
|
||||
return $this === self::Ai;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rules the AI categorizer is allowed to mutate or self-heal. A user's own
|
||||
* rules are sacred; a correction the user made is sacred too.
|
||||
*/
|
||||
public function isManagedByAi(): bool
|
||||
{
|
||||
return $this === self::Ai;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -213,13 +213,15 @@ class TransactionController extends Controller
|
|||
$hasLabelUpdate = $request->has('label_ids');
|
||||
unset($data['label_ids']);
|
||||
|
||||
// A user-set category overrides any AI assignment: log the correction,
|
||||
// self-heal the ai rule, and reset the provenance to manual.
|
||||
$learnedRule = null;
|
||||
|
||||
// A user-set category overrides any AI assignment: learn the correction as
|
||||
// a forward-looking rule, log/self-heal as needed, and reset provenance.
|
||||
if ($request->has('category_id')) {
|
||||
$newCategoryId = $data['category_id'] ?? null;
|
||||
|
||||
if ($newCategoryId !== $transaction->category_id) {
|
||||
app(CategoryOverrideHandler::class)->record($transaction, $newCategoryId);
|
||||
$learnedRule = app(CategoryOverrideHandler::class)->record($transaction, $newCategoryId);
|
||||
|
||||
$data['category_source'] = $newCategoryId === null ? null : CategorySource::Manual->value;
|
||||
$data['ai_confidence'] = null;
|
||||
|
|
@ -251,6 +253,11 @@ class TransactionController extends Controller
|
|||
|
||||
return response()->json([
|
||||
'data' => $transaction->fresh()->load('labels'),
|
||||
'learned_rule' => $learnedRule === null ? null : [
|
||||
'id' => $learnedRule->id,
|
||||
'title' => $learnedRule->title,
|
||||
'category_id' => $learnedRule->action_category_id,
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ use App\Enums\RuleOrigin;
|
|||
use App\Models\AutomationRule;
|
||||
use App\Models\Category;
|
||||
use App\Models\Transaction;
|
||||
use App\Services\Ai\Contracts\TransactionMatcher;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
/**
|
||||
|
|
@ -21,6 +22,11 @@ use Illuminate\Support\Str;
|
|||
*/
|
||||
class AiRuleLearner
|
||||
{
|
||||
public function __construct(
|
||||
private readonly DescriptionTokenizer $tokenizer,
|
||||
private readonly TransactionMatcher $matcher,
|
||||
) {}
|
||||
|
||||
public function learn(CategorizationOutcome $outcome): ?AutomationRule
|
||||
{
|
||||
if (! $outcome->merchantUnambiguous) {
|
||||
|
|
@ -50,6 +56,218 @@ class AiRuleLearner
|
|||
return $rule;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn a user's correction into a deterministic, forward-looking rule so the
|
||||
* same merchant (or the same distinctive description) is never mis-categorized
|
||||
* the same way again — the next matching transaction is categorized by this
|
||||
* rule before the model ever runs.
|
||||
*
|
||||
* The rule is keyed on the merchant when one exists (stable even as the
|
||||
* description varies); otherwise on the description's distinctive tokens,
|
||||
* guarded so an over-broad token can never silently mis-file en masse. A key
|
||||
* lives in exactly one correction rule, so changing your mind moves it.
|
||||
* Returns the rule that now carries the correction, or null when nothing safe
|
||||
* could be learned (correcting to uncategorized, no usable key, or guarded).
|
||||
*/
|
||||
public function learnFromCorrection(Transaction $transaction, ?string $toCategoryId): ?AutomationRule
|
||||
{
|
||||
if ($toCategoryId === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$clause = $this->correctionClause($transaction);
|
||||
|
||||
if ($clause === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$this->releaseClauseFromOtherCorrectionRules($transaction->user_id, $toCategoryId, $clause);
|
||||
|
||||
$rule = $this->existingCorrectionRule($transaction->user_id, $toCategoryId)
|
||||
?? $this->createCorrectionRule($transaction->user_id, $toCategoryId);
|
||||
|
||||
$this->appendClause($rule, $clause);
|
||||
|
||||
return $rule;
|
||||
}
|
||||
|
||||
/**
|
||||
* The JsonLogic clause that recognises future transactions like this one:
|
||||
* a merchant equality when a clean merchant key exists, otherwise an AND of
|
||||
* "description contains" over the distinctive tokens. Null when neither is
|
||||
* usable or the description token set is too broad to be safe.
|
||||
*
|
||||
* @return array<string, mixed>|null
|
||||
*/
|
||||
private function correctionClause(Transaction $transaction): ?array
|
||||
{
|
||||
$merchant = $this->merchantKey($transaction);
|
||||
|
||||
if ($merchant !== null) {
|
||||
[$field, $token] = $merchant;
|
||||
|
||||
return ['==' => [['var' => $field], $token]];
|
||||
}
|
||||
|
||||
return $this->descriptionClause($transaction);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>|null
|
||||
*/
|
||||
private function descriptionClause(Transaction $transaction): ?array
|
||||
{
|
||||
if ($transaction->description_iv !== null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$tokens = $this->distinctiveDescriptionTokens($transaction);
|
||||
|
||||
if ($tokens === []) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($this->isOverbroad($transaction, $tokens)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$clauses = array_map(
|
||||
fn (string $token): array => ['in' => [$token, ['var' => 'description']]],
|
||||
$tokens,
|
||||
);
|
||||
|
||||
return count($clauses) === 1 ? $clauses[0] : ['and' => $clauses];
|
||||
}
|
||||
|
||||
/**
|
||||
* The distinctive description tokens of this transaction relative to the
|
||||
* user's own transaction vocabulary (the noise corpus). Corrections are rare
|
||||
* and user-driven, so loading the descriptions here is acceptable.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function distinctiveDescriptionTokens(Transaction $transaction): array
|
||||
{
|
||||
$descriptions = Transaction::query()
|
||||
->where('user_id', $transaction->user_id)
|
||||
->whereNull('description_iv')
|
||||
->pluck('description')
|
||||
->all();
|
||||
|
||||
$frequency = $this->tokenizer->documentFrequency($descriptions);
|
||||
$threshold = count($descriptions) * (float) config('ai_suggestions.noise_token_fraction');
|
||||
|
||||
return $this->tokenizer->distinctiveTokens((string) $transaction->description, $frequency, $threshold);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a description rule over these tokens would match so many of the
|
||||
* user's uncategorized transactions that it risks mis-filing en masse.
|
||||
*
|
||||
* @param list<string> $tokens
|
||||
*/
|
||||
private function isOverbroad(Transaction $transaction, array $tokens): bool
|
||||
{
|
||||
$total = $this->matcher->total($transaction->user);
|
||||
|
||||
if ($total === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$conditions = array_map(
|
||||
fn (string $token): array => ['field' => 'description', 'operator' => 'contains', 'token' => $token],
|
||||
$tokens,
|
||||
);
|
||||
|
||||
$fraction = $this->matcher->countMatchingAll($transaction->user, $conditions) / $total;
|
||||
|
||||
return $fraction > (float) config('ai_suggestions.overbroad_fraction');
|
||||
}
|
||||
|
||||
private function existingCorrectionRule(string $userId, string $categoryId): ?AutomationRule
|
||||
{
|
||||
return AutomationRule::query()
|
||||
->where('user_id', $userId)
|
||||
->where('action_category_id', $categoryId)
|
||||
->origin(RuleOrigin::Correction)
|
||||
->first();
|
||||
}
|
||||
|
||||
private function createCorrectionRule(string $userId, string $categoryId): AutomationRule
|
||||
{
|
||||
// Appended at the bottom, like ai rules. A correction and an ai rule never
|
||||
// compete on the same key — forget() strips it from the ai rule the moment
|
||||
// the correction is made. ponytail: cross-key precedence is creation-order;
|
||||
// band correction above ai explicitly only if that ever bites.
|
||||
$priority = (int) AutomationRule::query()->where('user_id', $userId)->max('priority');
|
||||
|
||||
return AutomationRule::create([
|
||||
'user_id' => $userId,
|
||||
'title' => $this->title($categoryId, []),
|
||||
'priority' => $priority + 1,
|
||||
'origin' => RuleOrigin::Correction,
|
||||
'rules_json' => [],
|
||||
'action_category_id' => $categoryId,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep a key in exactly one correction rule: if the user re-corrects the same
|
||||
* merchant/description to a different category, drop the identical clause from
|
||||
* any other correction rule (deleting it when it becomes empty).
|
||||
*
|
||||
* @param array<string, mixed> $clause
|
||||
*/
|
||||
private function releaseClauseFromOtherCorrectionRules(string $userId, string $keepCategoryId, array $clause): void
|
||||
{
|
||||
$rules = AutomationRule::query()
|
||||
->where('user_id', $userId)
|
||||
->where('action_category_id', '!=', $keepCategoryId)
|
||||
->origin(RuleOrigin::Correction)
|
||||
->get();
|
||||
|
||||
foreach ($rules as $rule) {
|
||||
$clauses = $this->clauses($rule->rules_json);
|
||||
$remaining = array_values(array_filter($clauses, fn (array $existing): bool => $existing != $clause));
|
||||
|
||||
if (count($remaining) === count($clauses)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($remaining === []) {
|
||||
$rule->delete();
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$rule->rules_json = count($remaining) === 1 ? $remaining[0] : ['or' => $remaining];
|
||||
$rule->title = $this->title((string) $rule->action_category_id, $this->tokens($remaining));
|
||||
$rule->save();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a clause to a rule's OR set, skipping an identical existing one.
|
||||
*
|
||||
* @param array<string, mixed> $clause
|
||||
*/
|
||||
private function appendClause(AutomationRule $rule, array $clause): void
|
||||
{
|
||||
$clauses = $this->clauses($rule->rules_json);
|
||||
|
||||
foreach ($clauses as $existing) {
|
||||
if ($existing == $clause) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
$clauses[] = $clause;
|
||||
|
||||
$rule->rules_json = count($clauses) === 1 ? $clauses[0] : ['or' => $clauses];
|
||||
$rule->title = $this->title((string) $rule->action_category_id, $this->tokens($clauses));
|
||||
$rule->save();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0: string, 1: string}|null [field, token]
|
||||
*/
|
||||
|
|
@ -172,6 +390,10 @@ class AiRuleLearner
|
|||
}
|
||||
|
||||
/**
|
||||
* Human-readable token per clause, for the rule title: the value of a
|
||||
* merchant equality, the needle of a "contains", or the joined needles of an
|
||||
* AND-of-contains.
|
||||
*
|
||||
* @param list<array<string, mixed>> $clauses
|
||||
* @return list<string>
|
||||
*/
|
||||
|
|
@ -180,9 +402,9 @@ class AiRuleLearner
|
|||
$tokens = [];
|
||||
|
||||
foreach ($clauses as $clause) {
|
||||
$token = $clause['=='][1] ?? null;
|
||||
$token = $this->clauseLabel($clause);
|
||||
|
||||
if (is_string($token)) {
|
||||
if ($token !== null) {
|
||||
$tokens[] = $token;
|
||||
}
|
||||
}
|
||||
|
|
@ -190,6 +412,40 @@ class AiRuleLearner
|
|||
return $tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $clause
|
||||
*/
|
||||
private function clauseLabel(array $clause): ?string
|
||||
{
|
||||
if (isset($clause['=='])) {
|
||||
$token = $clause['=='][1] ?? null;
|
||||
|
||||
return is_string($token) ? $token : null;
|
||||
}
|
||||
|
||||
if (isset($clause['in'])) {
|
||||
$token = $clause['in'][0] ?? null;
|
||||
|
||||
return is_string($token) ? $token : null;
|
||||
}
|
||||
|
||||
if (isset($clause['and']) && is_array($clause['and'])) {
|
||||
$parts = [];
|
||||
|
||||
foreach ($clause['and'] as $sub) {
|
||||
$needle = $sub['in'][0] ?? null;
|
||||
|
||||
if (is_string($needle)) {
|
||||
$parts[] = $needle;
|
||||
}
|
||||
}
|
||||
|
||||
return $parts === [] ? null : implode(' + ', $parts);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<string> $tokens
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -10,46 +10,59 @@ use App\Models\Transaction;
|
|||
|
||||
/**
|
||||
* Runs when a user overrides a transaction's category. If the category being
|
||||
* replaced was assigned by AI — directly, or via an ai-owned rule — it records a
|
||||
* correction (the calibration signal) and self-heals the rule so the same wrong
|
||||
* category is not forced on future transactions from that merchant. User-owned
|
||||
* rules and manual categories are never touched.
|
||||
* replaced was assigned by the system — by AI directly, by an ai-owned rule, or
|
||||
* by a rule learned from an earlier correction — it learns the user's choice as
|
||||
* a deterministic, forward-looking rule so the same mistake is never repeated.
|
||||
*
|
||||
* For AI-driven corrections it also records the calibration signal and self-heals
|
||||
* the ai rule that mislabeled the merchant. User-owned rules, bank categories and
|
||||
* one-off manual categorizations are never learned from or touched.
|
||||
*
|
||||
* Must be called BEFORE the new category is written, while the transaction still
|
||||
* holds its previous categorization.
|
||||
* holds its previous categorization. Returns the rule that now carries the
|
||||
* correction (for an "undo" affordance), or null when nothing was learned.
|
||||
*/
|
||||
class CategoryOverrideHandler
|
||||
{
|
||||
public function __construct(private readonly AiRuleLearner $learner) {}
|
||||
|
||||
public function record(Transaction $transaction, ?string $newCategoryId): void
|
||||
public function record(Transaction $transaction, ?string $newCategoryId): ?AutomationRule
|
||||
{
|
||||
if ($newCategoryId === $transaction->category_id) {
|
||||
return;
|
||||
return null;
|
||||
}
|
||||
|
||||
$rule = $transaction->categorized_by_rule_id !== null
|
||||
? AutomationRule::query()->find($transaction->categorized_by_rule_id)
|
||||
: null;
|
||||
|
||||
$aiRule = $rule !== null && $rule->origin === RuleOrigin::Ai;
|
||||
$aiDriven = $transaction->category_source === CategorySource::Ai || $aiRule;
|
||||
$ruleOrigin = $rule?->origin;
|
||||
$aiDriven = $transaction->category_source === CategorySource::Ai || $ruleOrigin === RuleOrigin::Ai;
|
||||
$learnable = $aiDriven || $ruleOrigin === RuleOrigin::Correction;
|
||||
|
||||
if (! $aiDriven) {
|
||||
return;
|
||||
if (! $learnable) {
|
||||
return null;
|
||||
}
|
||||
|
||||
CategoryCorrection::create([
|
||||
'user_id' => $transaction->user_id,
|
||||
'transaction_id' => $transaction->id,
|
||||
'from_category_id' => $transaction->category_id,
|
||||
'to_category_id' => $newCategoryId,
|
||||
'source' => $transaction->category_source ?? CategorySource::Rule,
|
||||
'confidence' => $transaction->ai_confidence,
|
||||
]);
|
||||
// The correction signal calibrates AI accuracy, so only AI assignments
|
||||
// are logged — a correction rule overruling itself is not an AI miss.
|
||||
if ($aiDriven) {
|
||||
CategoryCorrection::create([
|
||||
'user_id' => $transaction->user_id,
|
||||
'transaction_id' => $transaction->id,
|
||||
'from_category_id' => $transaction->category_id,
|
||||
'to_category_id' => $newCategoryId,
|
||||
'source' => $transaction->category_source ?? CategorySource::Rule,
|
||||
'confidence' => $transaction->ai_confidence,
|
||||
]);
|
||||
}
|
||||
|
||||
if ($aiRule) {
|
||||
// Stop the ai rule from forcing the wrong category on this merchant again.
|
||||
// Correction rules self-correct instead, when the key is re-learned below.
|
||||
if ($ruleOrigin === RuleOrigin::Ai && $rule !== null) {
|
||||
$this->learner->forget($rule, $transaction);
|
||||
}
|
||||
|
||||
return $this->learner->learnFromCorrection($transaction, $newCategoryId);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,13 @@ interface TransactionMatcher
|
|||
*/
|
||||
public function countMatchingAny(User $user, array $conditions): int;
|
||||
|
||||
/**
|
||||
* Count uncategorized transactions matching ALL of the given conditions (AND).
|
||||
*
|
||||
* @param list<array{field: string, operator: string, token: string}> $conditions
|
||||
*/
|
||||
public function countMatchingAll(User $user, array $conditions): int;
|
||||
|
||||
/**
|
||||
* The uncategorized transactions matching ANY of the given conditions (OR).
|
||||
*
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Ai;
|
||||
|
||||
/**
|
||||
* Turns a free-text bank description into comparable, language-agnostic tokens
|
||||
* and isolates the distinctive ones — the part that stays constant across
|
||||
* "practically identical" variants of the same transaction. Structural noise
|
||||
* (words common across the corpus: "pago", "carte", a city, an operation type)
|
||||
* is dropped by document frequency rather than a hardcoded stopword list.
|
||||
*
|
||||
* Stateless and corpus-agnostic: callers supply the document frequency and the
|
||||
* noise threshold so the same logic serves both rule suggestions and learning
|
||||
* from a single corrected transaction.
|
||||
*/
|
||||
class DescriptionTokenizer
|
||||
{
|
||||
private const MIN_TOKEN_LENGTH = 3;
|
||||
|
||||
/**
|
||||
* Split a description into lowercased word tokens, with digits and
|
||||
* punctuation stripped and very short tokens dropped.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
public function tokens(string $value): array
|
||||
{
|
||||
$value = $this->normalize($value);
|
||||
$value = preg_replace('/[0-9]+/', ' ', $value) ?? $value;
|
||||
$value = preg_replace('/[^\p{L}\s]+/u', ' ', $value) ?? $value;
|
||||
$value = $this->normalize($value);
|
||||
|
||||
if ($value === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
return array_values(array_filter(
|
||||
explode(' ', $value),
|
||||
fn (string $token): bool => mb_strlen($token) >= self::MIN_TOKEN_LENGTH,
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Document frequency of each token across a corpus of descriptions: how many
|
||||
* descriptions each token appears in (counted once per description).
|
||||
*
|
||||
* @param iterable<int, string|null> $descriptions
|
||||
* @return array<string, int>
|
||||
*/
|
||||
public function documentFrequency(iterable $descriptions): array
|
||||
{
|
||||
$frequency = [];
|
||||
|
||||
foreach ($descriptions as $description) {
|
||||
foreach (array_unique($this->tokens((string) $description)) as $token) {
|
||||
$frequency[$token] = ($frequency[$token] ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
return $frequency;
|
||||
}
|
||||
|
||||
/**
|
||||
* The distinctive tokens of a description: those appearing in no more than
|
||||
* the noise threshold of the corpus, sorted for a stable key. If every token
|
||||
* is common (all noise), the full token set is kept as a fallback.
|
||||
*
|
||||
* @param array<string, int> $documentFrequency
|
||||
* @return list<string>
|
||||
*/
|
||||
public function distinctiveTokens(string $value, array $documentFrequency, float $noiseThreshold): array
|
||||
{
|
||||
$tokens = $this->tokens($value);
|
||||
|
||||
if ($tokens === []) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$distinctive = array_values(array_filter(
|
||||
$tokens,
|
||||
fn (string $token): bool => ($documentFrequency[$token] ?? 0) <= $noiseThreshold,
|
||||
));
|
||||
|
||||
if ($distinctive === []) {
|
||||
$distinctive = $tokens;
|
||||
}
|
||||
|
||||
$distinctive = array_values(array_unique($distinctive));
|
||||
sort($distinctive);
|
||||
|
||||
return $distinctive;
|
||||
}
|
||||
|
||||
/**
|
||||
* The distinctive tokens joined into a single stable grouping key.
|
||||
*
|
||||
* @param array<string, int> $documentFrequency
|
||||
*/
|
||||
public function distinctiveKey(string $value, array $documentFrequency, float $noiseThreshold): string
|
||||
{
|
||||
return implode(' ', $this->distinctiveTokens($value, $documentFrequency, $noiseThreshold));
|
||||
}
|
||||
|
||||
private function normalize(string $value): string
|
||||
{
|
||||
return trim(preg_replace('/\s+/', ' ', mb_strtolower($value)) ?? '');
|
||||
}
|
||||
}
|
||||
|
|
@ -19,7 +19,7 @@ class RuleSuggestionAggregator
|
|||
*/
|
||||
private const SAMPLE_LIMIT = 5;
|
||||
|
||||
private const MIN_TOKEN_LENGTH = 3;
|
||||
public function __construct(private readonly DescriptionTokenizer $tokenizer) {}
|
||||
|
||||
/**
|
||||
* Build the bounded set of transaction groups worth suggesting a rule for.
|
||||
|
|
@ -42,7 +42,7 @@ class RuleSuggestionAggregator
|
|||
foreach ($transactions as $transaction) {
|
||||
[$field, $rawKey] = $this->groupingSignal($transaction);
|
||||
$key = $field === 'description'
|
||||
? $this->distinctiveKey($rawKey, $documentFrequency, $noiseThreshold)
|
||||
? $this->tokenizer->distinctiveKey($rawKey, $documentFrequency, $noiseThreshold)
|
||||
: $this->normalizeWhitespace($rawKey);
|
||||
|
||||
if ($key === '') {
|
||||
|
|
@ -144,75 +144,13 @@ class RuleSuggestionAggregator
|
|||
*/
|
||||
private function descriptionDocumentFrequency(Collection $transactions): array
|
||||
{
|
||||
$frequency = [];
|
||||
$descriptions = $transactions
|
||||
->map(fn (Transaction $transaction): array => $this->groupingSignal($transaction))
|
||||
->filter(fn (array $signal): bool => $signal[0] === 'description')
|
||||
->map(fn (array $signal): string => $signal[1])
|
||||
->all();
|
||||
|
||||
foreach ($transactions as $transaction) {
|
||||
[$field, $rawKey] = $this->groupingSignal($transaction);
|
||||
|
||||
if ($field !== 'description') {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (array_unique($this->descriptionTokens($rawKey)) as $token) {
|
||||
$frequency[$token] = ($frequency[$token] ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
return $frequency;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a free-text description into comparable tokens (lowercased, digits
|
||||
* and punctuation stripped, very short tokens dropped).
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function descriptionTokens(string $value): array
|
||||
{
|
||||
$value = $this->normalizeWhitespace($value);
|
||||
$value = preg_replace('/[0-9]+/', ' ', $value) ?? $value;
|
||||
$value = preg_replace('/[^\p{L}\s]+/u', ' ', $value) ?? $value;
|
||||
$value = $this->normalizeWhitespace($value);
|
||||
|
||||
if ($value === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
return array_values(array_filter(
|
||||
explode(' ', $value),
|
||||
fn (string $token): bool => mb_strlen($token) >= self::MIN_TOKEN_LENGTH,
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a description's grouping key from only its distinctive tokens: words
|
||||
* appearing in more than the noise threshold of transactions are dropped so
|
||||
* variants of the same merchant collapse together. Language-agnostic — if
|
||||
* every token is common, the full token set is kept as a fallback.
|
||||
*
|
||||
* @param array<string, int> $documentFrequency
|
||||
*/
|
||||
private function distinctiveKey(string $value, array $documentFrequency, float $noiseThreshold): string
|
||||
{
|
||||
$tokens = $this->descriptionTokens($value);
|
||||
|
||||
if ($tokens === []) {
|
||||
return '';
|
||||
}
|
||||
|
||||
$distinctive = array_values(array_filter(
|
||||
$tokens,
|
||||
fn (string $token): bool => ($documentFrequency[$token] ?? 0) <= $noiseThreshold,
|
||||
));
|
||||
|
||||
if ($distinctive === []) {
|
||||
$distinctive = $tokens;
|
||||
}
|
||||
|
||||
$distinctive = array_values(array_unique($distinctive));
|
||||
sort($distinctive);
|
||||
|
||||
return implode(' ', $distinctive);
|
||||
return $this->tokenizer->documentFrequency($descriptions);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -69,6 +69,13 @@ class UncategorizedTransactionMatcher implements TransactionMatcher
|
|||
return $query->get();
|
||||
}
|
||||
|
||||
public function countMatchingAll(User $user, array $conditions): int
|
||||
{
|
||||
$query = $this->allQuery($user, $conditions);
|
||||
|
||||
return $query === null ? 0 : $query->count();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Builder<Transaction>
|
||||
*/
|
||||
|
|
@ -135,6 +142,44 @@ class UncategorizedTransactionMatcher implements TransactionMatcher
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a single query matching ALL of the conditions (AND). Invalid
|
||||
* conditions (unknown field or blank token) are skipped; returns null when
|
||||
* none remain.
|
||||
*
|
||||
* @param list<array{field: string, operator: string, token: string}> $conditions
|
||||
* @return Builder<Transaction>|null
|
||||
*/
|
||||
private function allQuery(User $user, array $conditions): ?Builder
|
||||
{
|
||||
$valid = array_values(array_filter(
|
||||
$conditions,
|
||||
fn (array $condition): bool => in_array($condition['field'], self::ALLOWED_FIELDS, true)
|
||||
&& trim($condition['token']) !== '',
|
||||
));
|
||||
|
||||
if ($valid === []) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$query = $this->baseQuery($user);
|
||||
|
||||
foreach ($valid as $condition) {
|
||||
$field = $condition['field'];
|
||||
$token = mb_strtolower(trim($condition['token']));
|
||||
|
||||
if ($condition['operator'] === 'equals') {
|
||||
$query->whereRaw("LOWER({$field}) = ?", [$token]);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$query->whereRaw("LOWER({$field}) LIKE ?", ['%'.$this->escapeLike($token).'%']);
|
||||
}
|
||||
|
||||
return $query;
|
||||
}
|
||||
|
||||
private function escapeLike(string $value): string
|
||||
{
|
||||
return str_replace(['\\', '%', '_'], ['\\\\', '\\%', '\\_'], $value);
|
||||
|
|
|
|||
|
|
@ -43,4 +43,14 @@ class AutomationRuleFactory extends Factory
|
|||
'origin' => RuleOrigin::Ai,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* A rule learned from a user's category correction.
|
||||
*/
|
||||
public function correction(): static
|
||||
{
|
||||
return $this->state(fn (array $attributes): array => [
|
||||
'origin' => RuleOrigin::Correction,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
{
|
||||
"Learned: similar transactions will be categorized automatically.": "Aprendido: las transacciones similares se categorizarán automáticamente.",
|
||||
"Undo": "Deshacer",
|
||||
"This subscription is no longer eligible for a self-service refund.": "Esta suscripción ya no es elegible para una devolución automática.",
|
||||
"Your payment was refunded, your subscription was canceled, and your bank connections were disconnected.": "Te hemos devuelto el pago, cancelado la suscripción y desconectado tus cuentas bancarias.",
|
||||
"Try it for :days days": "Pruébalo durante :days días",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { destroy } from '@/actions/App/Http/Controllers/Settings/AutomationRuleController';
|
||||
import { showsAiUpsell } from '@/components/transactions/ai-upsell-sample';
|
||||
import { CategorySelect } from '@/components/transactions/category-select';
|
||||
import { AiSparkleIcon } from '@/components/ui/ai-sparkle-icon';
|
||||
|
|
@ -19,6 +20,7 @@ import { type DecryptedTransaction } from '@/types/transaction';
|
|||
import { __ } from '@/utils/i18n';
|
||||
import { router, usePage } from '@inertiajs/react';
|
||||
import { useState } from 'react';
|
||||
import { toast } from 'sonner';
|
||||
|
||||
interface CategoryCellProps {
|
||||
transaction: DecryptedTransaction;
|
||||
|
|
@ -72,7 +74,10 @@ export function CategoryCell({
|
|||
category_id: categoryId,
|
||||
};
|
||||
|
||||
await transactionSyncService.update(transaction.id, updateData);
|
||||
const result = await transactionSyncService.update(
|
||||
transaction.id,
|
||||
updateData,
|
||||
);
|
||||
|
||||
const updatedCategory = categoryId
|
||||
? categories.find((c) => c.id === categoryId) || null
|
||||
|
|
@ -105,6 +110,30 @@ export function CategoryCell({
|
|||
'transaction_table',
|
||||
);
|
||||
}
|
||||
|
||||
// The correction taught the system a forward rule: similar
|
||||
// transactions are now categorized automatically. Offer an instant
|
||||
// undo in case the learned rule is broader than intended.
|
||||
if (result.learned_rule) {
|
||||
const ruleId = result.learned_rule.id;
|
||||
|
||||
toast.success(
|
||||
__(
|
||||
'Learned: similar transactions will be categorized automatically.',
|
||||
),
|
||||
{
|
||||
action: {
|
||||
label: __('Undo'),
|
||||
onClick: () => {
|
||||
router.delete(destroy(ruleId).url, {
|
||||
preserveScroll: true,
|
||||
preserveState: true,
|
||||
});
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to update category:', error);
|
||||
} finally {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,15 @@
|
|||
import { db } from '@/lib/dexie-db';
|
||||
import { TransactionSyncManager } from '@/lib/sync-manager';
|
||||
import type { LearnedRuleNotice } from '@/types/automation-rule';
|
||||
import type { Transaction } from '@/types/transaction';
|
||||
import type { UUID } from '@/types/uuid';
|
||||
import axios from 'axios';
|
||||
|
||||
/** A transaction update plus any rule the correction just taught the system. */
|
||||
export type UpdatedTransaction = Transaction & {
|
||||
learned_rule?: LearnedRuleNotice | null;
|
||||
};
|
||||
|
||||
interface TransactionUpdateData extends Partial<Transaction> {
|
||||
label_ids?: string[];
|
||||
}
|
||||
|
|
@ -96,7 +102,7 @@ class TransactionSyncService {
|
|||
async update(
|
||||
id: string,
|
||||
data: TransactionUpdateData,
|
||||
): Promise<Transaction> {
|
||||
): Promise<UpdatedTransaction> {
|
||||
const { label_ids, ...transactionData } = data;
|
||||
|
||||
const response = await axios.patch(`/transactions/${id}`, {
|
||||
|
|
@ -116,7 +122,8 @@ class TransactionSyncService {
|
|||
...restServerData,
|
||||
transaction_date: String(serverData.transaction_date).slice(0, 10),
|
||||
label_ids: serverLabelIds || [],
|
||||
} as Transaction;
|
||||
learned_rule: response.data.learned_rule ?? null,
|
||||
} as UpdatedTransaction;
|
||||
}
|
||||
|
||||
async updateMany(
|
||||
|
|
|
|||
|
|
@ -2,7 +2,17 @@ import type { Category } from './category';
|
|||
import type { Label } from './label';
|
||||
import { UUID } from './uuid';
|
||||
|
||||
export type RuleOrigin = 'user' | 'ai';
|
||||
export type RuleOrigin = 'user' | 'ai' | 'correction';
|
||||
|
||||
/**
|
||||
* A rule that was just learned from a category correction, returned by the
|
||||
* transaction update endpoint so the UI can offer an immediate undo.
|
||||
*/
|
||||
export interface LearnedRuleNotice {
|
||||
id: UUID;
|
||||
title: string;
|
||||
category_id: UUID | null;
|
||||
}
|
||||
|
||||
export interface AutomationRule {
|
||||
id: UUID;
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
use App\Enums\CategoryCashflowDirection;
|
||||
use App\Enums\CategorySource;
|
||||
use App\Enums\CategoryType;
|
||||
use App\Enums\RuleOrigin;
|
||||
use App\Models\AutomationRule;
|
||||
use App\Models\Category;
|
||||
use App\Models\CategoryCorrection;
|
||||
|
|
@ -153,3 +154,132 @@ it('does nothing when the category is unchanged', function () {
|
|||
|
||||
expect(CategoryCorrection::query()->count())->toBe(0);
|
||||
});
|
||||
|
||||
it('learns a forward rule from an ai correction so the next merchant transaction skips ai', function () {
|
||||
$user = User::factory()->create();
|
||||
$wrong = cohCategory($user);
|
||||
$right = cohCategory($user);
|
||||
|
||||
cohLearnRule($user, $wrong->id, 'Mercadona');
|
||||
$matched = cohMatched($user, 'Mercadona');
|
||||
|
||||
$learned = app(CategoryOverrideHandler::class)->record($matched, $right->id);
|
||||
|
||||
expect($learned)->not->toBeNull()
|
||||
->and($learned->origin)->toBe(RuleOrigin::Correction)
|
||||
->and($learned->action_category_id)->toBe($right->id)
|
||||
->and($learned->rules_json)->toBe(['==' => [['var' => 'creditor_name'], 'mercadona']]);
|
||||
|
||||
$next = cohMerchantTxn($user, 'Mercadona');
|
||||
app(AutomationRuleService::class)->applyRules($next);
|
||||
$next->refresh();
|
||||
|
||||
expect($next->category_id)->toBe($right->id)
|
||||
->and($next->category_source)->toBe(CategorySource::Rule)
|
||||
->and($next->categorized_by_rule_id)->toBe($learned->id);
|
||||
});
|
||||
|
||||
it('learns a description rule when there is no merchant key', function () {
|
||||
$user = User::factory()->create();
|
||||
$to = cohCategory($user);
|
||||
|
||||
$transaction = Transaction::factory()->plaintext()->create([
|
||||
'user_id' => $user->id,
|
||||
'category_id' => cohCategory($user)->id,
|
||||
'category_source' => CategorySource::Ai,
|
||||
'ai_confidence' => 0.9,
|
||||
'creditor_name' => null,
|
||||
'debtor_name' => null,
|
||||
'description' => 'Netflix subscription',
|
||||
]);
|
||||
|
||||
$learned = app(CategoryOverrideHandler::class)->record($transaction, $to->id);
|
||||
|
||||
expect($learned)->not->toBeNull()
|
||||
->and($learned->origin)->toBe(RuleOrigin::Correction)
|
||||
->and($learned->rules_json)->toBe([
|
||||
'and' => [
|
||||
['in' => ['netflix', ['var' => 'description']]],
|
||||
['in' => ['subscription', ['var' => 'description']]],
|
||||
],
|
||||
]);
|
||||
|
||||
$next = Transaction::factory()->plaintext()->create([
|
||||
'user_id' => $user->id,
|
||||
'category_id' => null,
|
||||
'creditor_name' => null,
|
||||
'debtor_name' => null,
|
||||
'description' => 'NETFLIX SUBSCRIPTION 12,99',
|
||||
]);
|
||||
app(AutomationRuleService::class)->applyRules($next);
|
||||
|
||||
expect($next->refresh()->category_id)->toBe($to->id);
|
||||
});
|
||||
|
||||
it('does not learn an over-broad description rule', function () {
|
||||
config()->set('ai_suggestions.overbroad_fraction', 0.1);
|
||||
|
||||
$user = User::factory()->create();
|
||||
$to = cohCategory($user);
|
||||
|
||||
foreach (range(1, 5) as $ignored) {
|
||||
Transaction::factory()->plaintext()->create([
|
||||
'user_id' => $user->id,
|
||||
'category_id' => null,
|
||||
'creditor_name' => null,
|
||||
'debtor_name' => null,
|
||||
'description' => 'Generic payment note',
|
||||
]);
|
||||
}
|
||||
|
||||
$transaction = Transaction::factory()->plaintext()->create([
|
||||
'user_id' => $user->id,
|
||||
'category_id' => cohCategory($user)->id,
|
||||
'category_source' => CategorySource::Ai,
|
||||
'ai_confidence' => 0.9,
|
||||
'creditor_name' => null,
|
||||
'debtor_name' => null,
|
||||
'description' => 'Generic payment note',
|
||||
]);
|
||||
|
||||
$learned = app(CategoryOverrideHandler::class)->record($transaction, $to->id);
|
||||
|
||||
expect($learned)->toBeNull()
|
||||
->and(AutomationRule::query()->origin(RuleOrigin::Correction)->count())->toBe(0);
|
||||
});
|
||||
|
||||
it('moves a merchant key to the new category when the user changes their mind', function () {
|
||||
$user = User::factory()->create();
|
||||
$first = cohCategory($user);
|
||||
$second = cohCategory($user);
|
||||
|
||||
cohLearnRule($user, cohCategory($user)->id, 'Mercadona');
|
||||
$matched = cohMatched($user, 'Mercadona');
|
||||
$firstRule = app(CategoryOverrideHandler::class)->record($matched, $first->id);
|
||||
|
||||
$later = cohMerchantTxn($user, 'Mercadona');
|
||||
app(AutomationRuleService::class)->applyRules($later);
|
||||
$later->refresh();
|
||||
|
||||
expect($later->categorized_by_rule_id)->toBe($firstRule->id);
|
||||
|
||||
$secondRule = app(CategoryOverrideHandler::class)->record($later, $second->id);
|
||||
|
||||
expect($secondRule->action_category_id)->toBe($second->id)
|
||||
->and($secondRule->rules_json)->toBe(['==' => [['var' => 'creditor_name'], 'mercadona']])
|
||||
->and(AutomationRule::query()->find($firstRule->id))->toBeNull()
|
||||
->and(CategoryCorrection::query()->count())->toBe(1);
|
||||
});
|
||||
|
||||
it('self-heals but learns nothing when correcting to uncategorized', function () {
|
||||
$user = User::factory()->create();
|
||||
cohLearnRule($user, cohCategory($user)->id, 'Mercadona');
|
||||
$matched = cohMatched($user, 'Mercadona');
|
||||
$aiRuleId = $matched->categorized_by_rule_id;
|
||||
|
||||
$learned = app(CategoryOverrideHandler::class)->record($matched, null);
|
||||
|
||||
expect($learned)->toBeNull()
|
||||
->and(AutomationRule::query()->find($aiRuleId))->toBeNull()
|
||||
->and(AutomationRule::query()->origin(RuleOrigin::Correction)->count())->toBe(0);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use App\Services\Ai\RuleSuggestionAggregator;
|
|||
beforeEach(function () {
|
||||
config()->set('ai_suggestions.min_group_count', 3);
|
||||
config()->set('ai_suggestions.max_groups_sent', 15);
|
||||
$this->aggregator = new RuleSuggestionAggregator;
|
||||
$this->aggregator = app(RuleSuggestionAggregator::class);
|
||||
});
|
||||
|
||||
function makeTxn(User $user, Account $account, array $attributes): void
|
||||
|
|
|
|||
Loading…
Reference in New Issue