From 291cfbe2612a3682f913216a64ebc76c699e55a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Falc=C3=B3n?= Date: Fri, 26 Jun 2026 14:53:55 +0200 Subject: [PATCH] feat(ai): record the model behind each AI categorization (#594) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why We pay for AI transaction categorization on Gemini, but we couldn't tell which model produced any given suggestion. The `gemini-flash-latest` alias used by categorization is **floating** — it silently drifted from the `flash-lite` tier up to `gemini-3.5-flash`, which is now the dominant cost on the Gemini dashboard. Before swapping categorization to a cheaper pinned model we need to measure accuracy per model, and for that we have to know which model labelled each row. We already store the AI-suggested category and its confidence on `transactions`; this adds the missing third axis: the model. ## What - New nullable `ai_model` column on `transactions`, written by `CategorizeTransactions::recordOutcome` from `config('ai_categorization.model')` whenever the model produces a suggestion (above or below the label bar). - The migration backfills existing AI-categorized rows (`ai_suggested_category_at` not null) with `gemini-3.5-flash` — the model `gemini-flash-latest` resolved to when those suggestions were made. Done as a raw update so it doesn't bump every row's `updated_at`. - `ai_model` added to `$fillable` and to `$hidden` (pure ops/measurement metadata, no need to ship it to the frontend). ## Testing - Extended `CategorizeTransactionsTest` to assert `ai_model` is stored on both the auto-applied and below-bar paths. - `php artisan test tests/Feature/Ai/CategorizeTransactionsTest.php` → 5 passed. ## Follow-ups (not in this PR) - Pin `AI_CATEGORIZATION_MODEL` to an explicit cheaper tier (mirror the existing `AI_SUGGESTIONS_MODEL=gemini-2.5-flash-lite`) and kill the floating alias in the config defaults. - Capture token usage from the SDK response for cost tracking. --- app/Models/Transaction.php | 3 ++ app/Services/Ai/CategorizeTransactions.php | 6 ++- ...609_add_ai_model_to_transactions_table.php | 38 +++++++++++++++++++ .../Feature/Ai/CategorizeTransactionsTest.php | 2 + 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 database/migrations/2026_06_26_124609_add_ai_model_to_transactions_table.php diff --git a/app/Models/Transaction.php b/app/Models/Transaction.php index 1273f4cb..12e3ebe6 100644 --- a/app/Models/Transaction.php +++ b/app/Models/Transaction.php @@ -29,6 +29,7 @@ use Illuminate\Database\Eloquent\SoftDeletes; * @property ?string $categorized_by_rule_id * @property ?string $ai_suggested_category_id * @property ?Carbon $ai_suggested_category_at + * @property ?string $ai_model */ class Transaction extends Model { @@ -51,6 +52,7 @@ class Transaction extends Model 'categorized_by_rule_id', 'ai_suggested_category_id', 'ai_suggested_category_at', + 'ai_model', 'description', 'description_iv', 'original_description', @@ -79,6 +81,7 @@ class Transaction extends Model 'dedup_fingerprint', 'raw_data', 'categorized_by_rule_id', + 'ai_model', 'deleted_at', ]; diff --git a/app/Services/Ai/CategorizeTransactions.php b/app/Services/Ai/CategorizeTransactions.php index b36c2137..fbae5fe8 100644 --- a/app/Services/Ai/CategorizeTransactions.php +++ b/app/Services/Ai/CategorizeTransactions.php @@ -42,6 +42,7 @@ class CategorizeTransactions $results = $this->resolve($transactions, $catalog); $labelBar = (float) config('ai_categorization.label_confidence'); + $model = (string) config('ai_categorization.model'); $outcomes = []; foreach ($results as $result) { @@ -62,7 +63,7 @@ class CategorizeTransactions $confidence = (float) ($result['confidence'] ?? 0.0); $applied = $confidence >= $labelBar; - $this->recordOutcome($transaction, $categoryId, $confidence, $applied); + $this->recordOutcome($transaction, $categoryId, $confidence, $applied, $model); $outcomes[] = new CategorizationOutcome( transaction: $transaction, @@ -82,11 +83,12 @@ class CategorizeTransactions * suggestion is kept (for confidence-bar tuning and a future confirm UI); * at or above it the category is also auto-applied. */ - private function recordOutcome(Transaction $transaction, string $categoryId, float $confidence, bool $applied): void + private function recordOutcome(Transaction $transaction, string $categoryId, float $confidence, bool $applied, string $model): void { $transaction->ai_suggested_category_id = $categoryId; $transaction->ai_confidence = $confidence; $transaction->ai_suggested_category_at = now(); + $transaction->ai_model = $model; if ($applied) { $transaction->category_id = $categoryId; diff --git a/database/migrations/2026_06_26_124609_add_ai_model_to_transactions_table.php b/database/migrations/2026_06_26_124609_add_ai_model_to_transactions_table.php new file mode 100644 index 00000000..625a6c98 --- /dev/null +++ b/database/migrations/2026_06_26_124609_add_ai_model_to_transactions_table.php @@ -0,0 +1,38 @@ +string('ai_model')->nullable()->after('ai_suggested_category_at'); + }); + + // Existing AI suggestions predate this column. They were produced by the + // floating `gemini-flash-latest` alias, which by then resolved to + // gemini-3.5-flash, so stamp that as the model used. + // ponytail: raw update so the backfill doesn't bump every row's updated_at. + DB::table('transactions') + ->whereNotNull('ai_suggested_category_at') + ->whereNull('ai_model') + ->update(['ai_model' => 'gemini-3.5-flash']); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('transactions', function (Blueprint $table) { + $table->dropColumn('ai_model'); + }); + } +}; diff --git a/tests/Feature/Ai/CategorizeTransactionsTest.php b/tests/Feature/Ai/CategorizeTransactionsTest.php index 08951f3c..bb4c9bf4 100644 --- a/tests/Feature/Ai/CategorizeTransactionsTest.php +++ b/tests/Feature/Ai/CategorizeTransactionsTest.php @@ -69,6 +69,7 @@ it('auto-applies the category when confidence clears the label bar', function () ->and($transaction->ai_confidence)->toEqual(0.95) ->and($transaction->ai_suggested_category_id)->toBe($category->id) ->and($transaction->ai_suggested_category_at)->not->toBeNull() + ->and($transaction->ai_model)->toBe((string) config('ai_categorization.model')) ->and($outcomes)->toHaveCount(1) ->and($outcomes[0]->applied)->toBeTrue() ->and($outcomes[0]->merchantUnambiguous)->toBeTrue(); @@ -99,6 +100,7 @@ it('leaves the transaction blank but records the suggestion when confidence is b ->and($transaction->ai_suggested_category_id)->toBe($category->id) ->and($transaction->ai_confidence)->toEqual(0.5) ->and($transaction->ai_suggested_category_at)->not->toBeNull() + ->and($transaction->ai_model)->toBe((string) config('ai_categorization.model')) ->and($outcomes)->toHaveCount(1) ->and($outcomes[0]->applied)->toBeFalse(); });