From 2472260ec88ff80c4215d7f37db2f4e591608304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vi=CC=81ctor=20Falco=CC=81n?= Date: Fri, 17 Jul 2026 18:43:58 +0200 Subject: [PATCH] feat(ai): make AI categorization confidence threshold configurable The confidence bar that decides whether the AI's suggested category is auto-applied to a transaction was a fixed config value (ai_categorization.label_confidence, 0.7). Let each user tune it with a slider in Settings > Manage Plan, next to the AI categorization toggle. - Add nullable user_settings.ai_confidence_threshold (percent, 0-100); null falls back to the global config default. - User::aiConfidenceThresholdPercent() centralises the "user value else config default" resolution used by both the billing page and the categorization engine. - CategorizeTransactions reads the per-user bar instead of the config. - Gate tier-2 rule learning on the outcome being applied, so the effective rule bar becomes max(label bar, rule_confidence): raising the threshold to reduce automation now also holds back forward rules (previously a suggestion above 0.85 could still spawn a rule even when the user's raised bar left the transaction uncategorized). - Slider (50-95%, debounced save) rendered when AI consent is on. --- .../AiConfidenceThresholdController.php | 20 +++++ .../Controllers/SubscriptionController.php | 1 + .../UpdateAiConfidenceThresholdRequest.php | 29 +++++++ app/Models/User.php | 11 +++ app/Models/UserSetting.php | 3 + app/Services/Ai/AiRuleLearner.php | 8 ++ app/Services/Ai/CategorizeTransactions.php | 2 +- ...dence_threshold_to_user_settings_table.php | 28 +++++++ lang/es.json | 2 + resources/js/pages/settings/billing.tsx | 78 ++++++++++++++++++- routes/settings.php | 4 + tests/Feature/Ai/AiRuleLearnerTest.php | 13 ++++ .../Feature/Ai/CategorizeTransactionsTest.php | 26 +++++++ tests/Feature/AiConsentSettingsTest.php | 17 ++++ .../Settings/AiConfidenceThresholdTest.php | 70 +++++++++++++++++ 15 files changed, 308 insertions(+), 4 deletions(-) create mode 100644 app/Http/Controllers/Settings/AiConfidenceThresholdController.php create mode 100644 app/Http/Requests/Settings/UpdateAiConfidenceThresholdRequest.php create mode 100644 database/migrations/2026_07_17_000000_add_ai_confidence_threshold_to_user_settings_table.php create mode 100644 tests/Feature/Settings/AiConfidenceThresholdTest.php diff --git a/app/Http/Controllers/Settings/AiConfidenceThresholdController.php b/app/Http/Controllers/Settings/AiConfidenceThresholdController.php new file mode 100644 index 00000000..b4fd2afb --- /dev/null +++ b/app/Http/Controllers/Settings/AiConfidenceThresholdController.php @@ -0,0 +1,20 @@ +user()->setting()->updateOrCreate( + ['user_id' => $request->user()->id], + ['ai_confidence_threshold' => $request->integer('ai_confidence_threshold')] + ); + + return back(); + } +} diff --git a/app/Http/Controllers/SubscriptionController.php b/app/Http/Controllers/SubscriptionController.php index 39e6db5f..b2656039 100644 --- a/app/Http/Controllers/SubscriptionController.php +++ b/app/Http/Controllers/SubscriptionController.php @@ -191,6 +191,7 @@ class SubscriptionController extends Controller return Inertia::render('settings/billing', [ 'hasAiConsent' => $user->hasActiveAiConsent(), + 'aiConfidenceThreshold' => $user->aiConfidenceThresholdPercent(), 'refund' => [ 'canSelfRefund' => $this->experimentOffer->canSelfRefund($user), 'deadline' => $subscription !== null && $this->experimentOffer->variantFor($user) === SubscriptionExperiment::PAY_NOW diff --git a/app/Http/Requests/Settings/UpdateAiConfidenceThresholdRequest.php b/app/Http/Requests/Settings/UpdateAiConfidenceThresholdRequest.php new file mode 100644 index 00000000..445bbba0 --- /dev/null +++ b/app/Http/Requests/Settings/UpdateAiConfidenceThresholdRequest.php @@ -0,0 +1,29 @@ +|string> + */ + public function rules(): array + { + return [ + 'ai_confidence_threshold' => ['required', 'integer', 'min:50', 'max:95'], + ]; + } +} diff --git a/app/Models/User.php b/app/Models/User.php index 2b09ecc8..d89dc8f7 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -285,6 +285,17 @@ class User extends Authenticatable implements HasLocalePreference, MustVerifyEma return $this->aiConsents()->active($scope)->exists(); } + /** + * The minimum confidence (as a percentage, 0-100) at which an AI-suggested + * category is auto-applied, falling back to the global default when the user + * has never adjusted it. Stored in percent; the engine divides by 100. + */ + public function aiConfidenceThresholdPercent(): int + { + return $this->setting->ai_confidence_threshold + ?? (int) round((float) config('ai_categorization.label_confidence') * 100); + } + /** * Record an AI consent for the current consent version (idempotent). */ diff --git a/app/Models/UserSetting.php b/app/Models/UserSetting.php index 0e24777c..9428f677 100644 --- a/app/Models/UserSetting.php +++ b/app/Models/UserSetting.php @@ -14,6 +14,7 @@ use Illuminate\Database\Eloquent\Relations\BelongsTo; * @property bool $include_loans_in_net_worth_chart * @property bool $include_real_estate_in_net_worth_chart * @property bool $notify_on_bank_transactions_synced + * @property ?int $ai_confidence_threshold */ class UserSetting extends Model { @@ -26,6 +27,7 @@ class UserSetting extends Model 'include_loans_in_net_worth_chart', 'include_real_estate_in_net_worth_chart', 'notify_on_bank_transactions_synced', + 'ai_confidence_threshold', ]; protected function casts(): array @@ -35,6 +37,7 @@ class UserSetting extends Model 'include_loans_in_net_worth_chart' => 'boolean', 'include_real_estate_in_net_worth_chart' => 'boolean', 'notify_on_bank_transactions_synced' => 'boolean', + 'ai_confidence_threshold' => 'integer', ]; } diff --git a/app/Services/Ai/AiRuleLearner.php b/app/Services/Ai/AiRuleLearner.php index f29cb515..6e8dc222 100644 --- a/app/Services/Ai/AiRuleLearner.php +++ b/app/Services/Ai/AiRuleLearner.php @@ -64,6 +64,14 @@ class AiRuleLearner return null; } + // Never generalise a suggestion we weren't confident enough to even apply + // to the single transaction. This ties tier 2 to the user's (possibly + // raised) label bar: the effective rule bar is max(label bar, rule bar), + // so raising the threshold to reduce automation also holds back rules. + if (! $outcome->applied) { + return null; + } + if ($outcome->confidence < (float) config('ai_categorization.rule_confidence')) { return null; } diff --git a/app/Services/Ai/CategorizeTransactions.php b/app/Services/Ai/CategorizeTransactions.php index 674b3b85..deb5f1b9 100644 --- a/app/Services/Ai/CategorizeTransactions.php +++ b/app/Services/Ai/CategorizeTransactions.php @@ -44,7 +44,7 @@ class CategorizeTransactions $byRef = $transactions->keyBy(fn (Transaction $transaction): string => $transaction->id); $results = $this->resolve($user, $transactions, $catalog); - $labelBar = (float) config('ai_categorization.label_confidence'); + $labelBar = $user->aiConfidenceThresholdPercent() / 100; $model = (string) config('ai_categorization.model'); $outcomes = []; diff --git a/database/migrations/2026_07_17_000000_add_ai_confidence_threshold_to_user_settings_table.php b/database/migrations/2026_07_17_000000_add_ai_confidence_threshold_to_user_settings_table.php new file mode 100644 index 00000000..722a7fd3 --- /dev/null +++ b/database/migrations/2026_07_17_000000_add_ai_confidence_threshold_to_user_settings_table.php @@ -0,0 +1,28 @@ +unsignedTinyInteger('ai_confidence_threshold')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('user_settings', function (Blueprint $table) { + $table->dropColumn('ai_confidence_threshold'); + }); + } +}; diff --git a/lang/es.json b/lang/es.json index a61252f1..50a18bd3 100644 --- a/lang/es.json +++ b/lang/es.json @@ -22,6 +22,8 @@ "With your permission, we send merchant names from your transactions to our AI provider so it can suggest categories. You can revoke this at any time.": "Con tu permiso, enviamos los nombres de los comercios de tus transacciones a nuestro proveedor de IA para que pueda sugerir categorías. Puedes revocarlo en cualquier momento.", "AI categorization enabled": "Categorización con IA activada", "AI categorization disabled": "Categorización con IA desactivada", + "Confidence threshold": "Umbral de confianza", + "Only auto-apply a suggested category when the AI is at least this confident. Higher keeps more transactions uncategorized but makes the ones it does apply more accurate. Changes apply to transactions categorized from now on.": "Aplica automáticamente una categoría sugerida solo cuando la IA tenga al menos esta confianza. Un valor más alto deja más transacciones sin categorizar, pero hace que las que sí se aplican sean más precisas. Los cambios se aplican a las transacciones categorizadas a partir de ahora.", "AI enabled. You can change this anytime in Settings > Manage Plan.": "IA activada. Puedes cambiar esta opción cuando quieras en Configuración > Gestionar Plan.", "AI not enabled. You can turn it on anytime in Settings > Manage Plan.": "IA no activada. Puedes activarla cuando quieras en Configuración > Gestionar Plan.", "Dismiss": "Descartar", diff --git a/resources/js/pages/settings/billing.tsx b/resources/js/pages/settings/billing.tsx index f28b7685..556b7fd6 100644 --- a/resources/js/pages/settings/billing.tsx +++ b/resources/js/pages/settings/billing.tsx @@ -29,7 +29,7 @@ import { SparklesIcon, ZapIcon, } from 'lucide-react'; -import { useState } from 'react'; +import { useRef, useState } from 'react'; import { toast } from 'sonner'; const breadcrumbs: BreadcrumbItem[] = [ @@ -389,12 +389,66 @@ function SubscribedSection({ ); } +function ConfidenceThresholdControl({ initial }: { initial: number }) { + const [value, setValue] = useState(initial); + const saveTimer = useRef>(undefined); + + const handleChange = (next: number) => { + setValue(next); + clearTimeout(saveTimer.current); + saveTimer.current = setTimeout(() => { + router.patch( + '/settings/ai-confidence-threshold', + { ai_confidence_threshold: next }, + { + preserveScroll: true, + preserveState: true, + only: ['aiConfidenceThreshold'], + }, + ); + }, 400); + }; + + return ( +
+
+ + {__('Confidence threshold')} + + + {value}% + +
+

+ {__( + 'Only auto-apply a suggested category when the AI is at least this confident. Higher keeps more transactions uncategorized but makes the ones it does apply more accurate. Changes apply to transactions categorized from now on.', + )} +

+ + handleChange(Number(event.currentTarget.value)) + } + className="mt-3 w-full accent-emerald-600" + /> +
+ ); +} + function AiConsentSection({ initialConsent, hasProPlan, + initialConfidenceThreshold, }: { initialConsent: boolean; hasProPlan: boolean; + initialConfidenceThreshold: number; }) { const [consented, setConsented] = useState(initialConsent); const [saving, setSaving] = useState(false); @@ -458,6 +512,12 @@ function AiConsentSection({ + {consented && ( + + )} + {!hasProPlan && (

{__( @@ -471,8 +531,19 @@ function AiConsentSection({ } export default function Billing() { - const { auth, pricing, locale, hasAiConsent, refund } = usePage< - SharedData & { hasAiConsent: boolean; refund?: RefundInfo } + const { + auth, + pricing, + locale, + hasAiConsent, + aiConfidenceThreshold, + refund, + } = usePage< + SharedData & { + hasAiConsent: boolean; + aiConfidenceThreshold: number; + refund?: RefundInfo; + } >().props; const isDemoAccount = auth?.isDemoAccount ?? false; const hasProPlan = auth?.hasProPlan ?? false; @@ -505,6 +576,7 @@ export default function Billing() { diff --git a/routes/settings.php b/routes/settings.php index 18b67346..728fd90c 100644 --- a/routes/settings.php +++ b/routes/settings.php @@ -2,6 +2,7 @@ use App\Http\Controllers\OpenBanking\ConnectionController; use App\Http\Controllers\Settings\AccountController; +use App\Http\Controllers\Settings\AiConfidenceThresholdController; use App\Http\Controllers\Settings\AutomationRuleApplicationController; use App\Http\Controllers\Settings\AutomationRuleController; use App\Http\Controllers\Settings\BankController; @@ -97,6 +98,9 @@ Route::middleware('auth')->group(function () { Route::patch('settings/net-worth-chart-real-estate-preference', [NetWorthChartRealEstatePreferenceController::class, 'update']) ->name('net-worth-chart-real-estate-preference.update'); + Route::patch('settings/ai-confidence-threshold', [AiConfidenceThresholdController::class, 'update']) + ->name('ai-confidence-threshold.update'); + Route::get('settings/billing', [SubscriptionController::class, 'billing'])->name('settings.billing'); Route::get('settings/billing/portal', [SubscriptionController::class, 'billingPortal'])->name('settings.billing.portal'); Route::post('settings/billing/refund', [SubscriptionController::class, 'refund']) diff --git a/tests/Feature/Ai/AiRuleLearnerTest.php b/tests/Feature/Ai/AiRuleLearnerTest.php index 1119579b..a971ba46 100644 --- a/tests/Feature/Ai/AiRuleLearnerTest.php +++ b/tests/Feature/Ai/AiRuleLearnerTest.php @@ -52,6 +52,19 @@ it('creates an ai-owned rule at the lowest priority and links the transaction', ->and($transaction->refresh()->categorized_by_rule_id)->toBe($rule->id); }); +it('does not learn a rule from a confident suggestion that was not applied', function () { + $user = User::factory()->create(); + $category = expenseCategory($user); + $transaction = merchantTransaction($user, 'Mercadona'); + + // Above rule_confidence and unambiguous, but the user's raised label bar kept + // it from being applied (applied = false) — no forward rule should be learned. + $notApplied = new CategorizationOutcome($transaction, $category->id, 0.9, true, false); + + expect(app(AiRuleLearner::class)->learn($notApplied))->toBeNull() + ->and($transaction->refresh()->categorized_by_rule_id)->toBeNull(); +}); + it('resolves a fresh instance per container lookup so the memoized corpus cannot leak or go stale', function () { // The per-user corpus cache has no invalidation and is safe only while the // learner is never a singleton. Guard that invariant. diff --git a/tests/Feature/Ai/CategorizeTransactionsTest.php b/tests/Feature/Ai/CategorizeTransactionsTest.php index b25a3993..41103bb6 100644 --- a/tests/Feature/Ai/CategorizeTransactionsTest.php +++ b/tests/Feature/Ai/CategorizeTransactionsTest.php @@ -79,6 +79,32 @@ it('auto-applies the category when confidence clears the label bar', function () ->and($outcomes[0]->merchantUnambiguous)->toBeTrue(); }); +it('honours the user confidence threshold over the config default', function () { + $user = User::factory()->create(); + $user->setting()->create(['ai_confidence_threshold' => 90]); + $category = groceries($user); + $transaction = uncategorized($user); + + $index = leafIndex(CategoryCatalog::forUser($user), $category->id); + + TransactionCategorizationAgent::fake([ + ['results' => [[ + 'ref' => $transaction->id, + 'category_index' => $index, + 'confidence' => 0.8, + 'merchant_unambiguous' => true, + ]]], + ]); + + $outcomes = app(CategorizeTransactions::class)->forTransactions($user, collect([$transaction])); + + $transaction->refresh(); + + expect($transaction->category_id)->toBeNull() + ->and($transaction->ai_suggested_category_id)->toBe($category->id) + ->and($outcomes[0]->applied)->toBeFalse(); +}); + it('leaves the transaction blank but records the suggestion when confidence is below the label bar', function () { $user = User::factory()->create(); $category = groceries($user); diff --git a/tests/Feature/AiConsentSettingsTest.php b/tests/Feature/AiConsentSettingsTest.php index e7edacf5..3525731a 100644 --- a/tests/Feature/AiConsentSettingsTest.php +++ b/tests/Feature/AiConsentSettingsTest.php @@ -23,6 +23,23 @@ test('billing page reports current ai consent state', function () { ); }); +test('billing page reports the ai confidence threshold, defaulting to the config bar', function () { + config(['subscriptions.enabled' => true, 'ai_categorization.label_confidence' => 0.7]); + $user = User::factory()->onboarded()->create(); + + actingAs($user)->withoutVite()->get(route('settings.billing')) + ->assertInertia(fn (Assert $page) => $page + ->where('aiConfidenceThreshold', 70) + ); + + $user->setting()->create(['ai_confidence_threshold' => 85]); + + actingAs($user->fresh())->withoutVite()->get(route('settings.billing')) + ->assertInertia(fn (Assert $page) => $page + ->where('aiConfidenceThreshold', 85) + ); +}); + test('transactions page reports current ai consent state', function () { $user = User::factory()->onboarded()->create(); diff --git a/tests/Feature/Settings/AiConfidenceThresholdTest.php b/tests/Feature/Settings/AiConfidenceThresholdTest.php new file mode 100644 index 00000000..377443ec --- /dev/null +++ b/tests/Feature/Settings/AiConfidenceThresholdTest.php @@ -0,0 +1,70 @@ + false]); +}); + +test('ai confidence threshold can be updated', function () { + $user = User::factory()->create(); + + $response = $this + ->actingAs($user) + ->patch(route('ai-confidence-threshold.update'), [ + 'ai_confidence_threshold' => 85, + ]); + + $response->assertSessionHasNoErrors()->assertRedirect(); + + expect($user->fresh()->setting->ai_confidence_threshold)->toBe(85); +}); + +test('ai confidence threshold rejects values out of range', function () { + $user = User::factory()->create(); + + $this + ->actingAs($user) + ->patch(route('ai-confidence-threshold.update'), [ + 'ai_confidence_threshold' => 10, + ]) + ->assertSessionHasErrors('ai_confidence_threshold'); + + $this + ->actingAs($user) + ->patch(route('ai-confidence-threshold.update'), [ + 'ai_confidence_threshold' => 100, + ]) + ->assertSessionHasErrors('ai_confidence_threshold'); +}); + +test('ai confidence threshold rejects non-integer values', function () { + $user = User::factory()->create(); + + $this + ->actingAs($user) + ->patch(route('ai-confidence-threshold.update'), [ + 'ai_confidence_threshold' => 'high', + ]) + ->assertSessionHasErrors('ai_confidence_threshold'); +}); + +test('ai confidence threshold requires authentication', function () { + $this->patch(route('ai-confidence-threshold.update'), [ + 'ai_confidence_threshold' => 80, + ])->assertRedirect(route('register')); +}); + +test('ai confidence threshold creates setting when none exists', function () { + $user = User::factory()->create(); + + expect(UserSetting::where('user_id', $user->id)->exists())->toBeFalse(); + + $this->actingAs($user) + ->patch(route('ai-confidence-threshold.update'), [ + 'ai_confidence_threshold' => 60, + ]); + + expect($user->fresh()->setting->ai_confidence_threshold)->toBe(60); +});