From 4038e60fbcd9891ceecabe5f66c34dde1abcc6cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Falc=C3=B3n?= Date: Fri, 26 Jun 2026 20:07:06 +0200 Subject: [PATCH] =?UTF-8?q?fix(ai):=20handle=20transient=20AI=20provider?= =?UTF-8?q?=20overloads=20=E2=80=94=20stop=20the=20Sentry=20noise=20and=20?= =?UTF-8?q?retry=20the=20dropped=20work=20(#595)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Sentry issue **PHP-LARAVEL-3S** (`ProviderOverloadedException: AI provider [gemini] is overloaded`) recurs whenever Gemini returns 503/429 under high demand — 7 events in 3h during the last surge, 0 users impacted. Two problems behind it: 1. **Sentry noise.** `CategorizeTransactions::resolveChunkWithRetry` retries, `laravel/ai` fails over providers, and a still-failing chunk is deliberately dropped so the rest of the backfill proceeds. But the catch reported **every** dropped chunk via `report()`, so an expected, self-healing transient condition floods Sentry and buries real bugs. 2. **Silently lost work.** A dropped chunk leaves those transactions uncategorized (`category_id NULL`) with nothing to re-trigger them: the backfill jobs are one-shot (`tries = 1`), there is no scheduled backfill, and the real-time listener only handles *new* transactions. They stay uncategorized until someone manually re-runs a backfill. ## What **1. Stop reporting transient failures.** Catch `FailoverableException` (the marker interface for `ProviderOverloadedException` / `RateLimitedException`) separately and log a warning instead of reporting it. Everything else still goes to `report()` unchanged, so real failures (malformed responses, insufficient credits, …) keep surfacing. **2. Retry the dropped work.** On a transient failure, schedule a deferred, per-user `RetryTransientAiCategorizationJob` that re-reads the user's still-pending transactions once the provider has had time to recover (`ai_categorization.retry_delay`, default 10 min). - `ShouldBeUnique` per user collapses a surge of dropped chunks into a **single** retry. - The unique lock is held through processing, so a retry that overloads again **cannot chain another** — exactly one deferred attempt per failure, no infinite loop. Failed 503s aren't billed. - Wired in `resolve()`, so it covers every entry point (backfill, onboarding, real-time listener, admin command). - Model cost is negligible (per config), so the retry re-runs the existing backfill rather than tracking which exact chunks failed. ## Tests - Transient overload → chunk dropped, nothing reported, retry scheduled for the user. - Unexpected failure → reported, **no** retry scheduled. - Retry job → categorizes still-pending transactions for a consenting user; no-op without consent (agent never prompted). Full `tests/Feature/Ai` + listeners suite green (111 tests); Pint clean. ## Existing prod backlog The transactions already dropped before this ships stay `pending`; an `ai:categorize-backfill ` recovers them on demand. Fixes PHP-LARAVEL-3S --- .../RetryTransientAiCategorizationJob.php | 58 ++++++++++++++ app/Services/Ai/CategorizeTransactions.php | 17 ++++- config/ai_categorization.php | 13 ++++ .../Feature/Ai/CategorizeTransactionsTest.php | 46 +++++++++++ .../RetryTransientAiCategorizationJobTest.php | 76 +++++++++++++++++++ 5 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 app/Jobs/RetryTransientAiCategorizationJob.php create mode 100644 tests/Feature/Ai/RetryTransientAiCategorizationJobTest.php diff --git a/app/Jobs/RetryTransientAiCategorizationJob.php b/app/Jobs/RetryTransientAiCategorizationJob.php new file mode 100644 index 00000000..d0e99388 --- /dev/null +++ b/app/Jobs/RetryTransientAiCategorizationJob.php @@ -0,0 +1,58 @@ +user->id; + } + + public function handle(AiCategorizationGate $gate, AiCategorizer $categorizer): void + { + if (! $gate->allows($this->user)) { + return; + } + + $categorizer->backfill($this->user); + } +} diff --git a/app/Services/Ai/CategorizeTransactions.php b/app/Services/Ai/CategorizeTransactions.php index fbae5fe8..674b3b85 100644 --- a/app/Services/Ai/CategorizeTransactions.php +++ b/app/Services/Ai/CategorizeTransactions.php @@ -4,10 +4,13 @@ namespace App\Services\Ai; use App\Ai\Agents\TransactionCategorizationAgent; use App\Enums\CategorySource; +use App\Jobs\RetryTransientAiCategorizationJob; use App\Models\Transaction; use App\Models\User; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Log; use Laravel\Ai\Enums\Lab; +use Laravel\Ai\Exceptions\FailoverableException; use Throwable; /** @@ -39,7 +42,7 @@ class CategorizeTransactions } $byRef = $transactions->keyBy(fn (Transaction $transaction): string => $transaction->id); - $results = $this->resolve($transactions, $catalog); + $results = $this->resolve($user, $transactions, $catalog); $labelBar = (float) config('ai_categorization.label_confidence'); $model = (string) config('ai_categorization.model'); @@ -101,12 +104,13 @@ class CategorizeTransactions /** * Send the transactions to the model in bounded chunks and merge the * results. A chunk that fails after a retry is dropped without discarding - * the chunks that succeeded. + * the chunks that succeeded; a transient provider failure additionally + * schedules a deferred retry of the user's still-pending transactions. * * @param Collection $transactions * @return list> */ - private function resolve(Collection $transactions, CategoryCatalog $catalog): array + private function resolve(User $user, Collection $transactions, CategoryCatalog $catalog): array { $batchSize = max(1, (int) config('ai_categorization.group_batch_size')); $results = []; @@ -116,6 +120,13 @@ class CategorizeTransactions foreach ($this->resolveChunkWithRetry($chunk, $catalog) as $result) { $results[] = $result; } + } catch (FailoverableException $exception) { + Log::warning('AI categorization chunk dropped: provider transient failure.', [ + 'exception' => $exception->getMessage(), + ]); + + RetryTransientAiCategorizationJob::dispatch($user) + ->delay(now()->addMinutes((int) config('ai_categorization.retry_delay'))); } catch (Throwable $exception) { report($exception); } diff --git a/config/ai_categorization.php b/config/ai_categorization.php index 78d9f7bd..17b9e580 100644 --- a/config/ai_categorization.php +++ b/config/ai_categorization.php @@ -69,6 +69,19 @@ return [ 'queue' => env('AI_CATEGORIZATION_QUEUE', 'ai'), + /* + |-------------------------------------------------------------------------- + | Transient-failure retry delay + |-------------------------------------------------------------------------- + | + | Minutes to wait before retrying a user's still-pending transactions after + | the AI provider dropped a chunk with a transient failure (overload / rate + | limit). The delay lets the provider recover before we try again. + | + */ + + 'retry_delay' => (int) env('AI_CATEGORIZATION_RETRY_DELAY', 10), + /* |-------------------------------------------------------------------------- | Free-plan upsell nudge diff --git a/tests/Feature/Ai/CategorizeTransactionsTest.php b/tests/Feature/Ai/CategorizeTransactionsTest.php index bb4c9bf4..b25a3993 100644 --- a/tests/Feature/Ai/CategorizeTransactionsTest.php +++ b/tests/Feature/Ai/CategorizeTransactionsTest.php @@ -4,11 +4,15 @@ use App\Ai\Agents\TransactionCategorizationAgent; use App\Enums\CategoryCashflowDirection; use App\Enums\CategorySource; use App\Enums\CategoryType; +use App\Jobs\RetryTransientAiCategorizationJob; use App\Models\Category; use App\Models\Transaction; use App\Models\User; use App\Services\Ai\CategorizeTransactions; use App\Services\Ai\CategoryCatalog; +use Illuminate\Support\Facades\Exceptions; +use Illuminate\Support\Facades\Queue; +use Laravel\Ai\Exceptions\ProviderOverloadedException; function leafIndex(CategoryCatalog $catalog, string $categoryId): int { @@ -132,6 +136,48 @@ it('never sends client-side encrypted transactions to the model', function () { ->and($encrypted->category_id)->toBeNull(); }); +it('drops the chunk, skips reporting and schedules a retry when the provider is transiently overloaded', function () { + $user = User::factory()->create(); + groceries($user); + $transaction = uncategorized($user); + + Exceptions::fake(); + Queue::fake(); + + TransactionCategorizationAgent::fake(fn () => throw ProviderOverloadedException::forProvider('gemini')); + + $outcomes = app(CategorizeTransactions::class)->forTransactions($user, collect([$transaction])); + + $transaction->refresh(); + + expect($outcomes)->toBe([]) + ->and($transaction->category_id)->toBeNull(); + + Exceptions::assertNothingReported(); + Queue::assertPushed( + RetryTransientAiCategorizationJob::class, + fn (RetryTransientAiCategorizationJob $job): bool => $job->user->is($user), + ); +}); + +it('reports unexpected failures and does not schedule a retry so real bugs are not swallowed', function () { + $user = User::factory()->create(); + groceries($user); + $transaction = uncategorized($user); + + Exceptions::fake(); + Queue::fake(); + + TransactionCategorizationAgent::fake(fn () => throw new RuntimeException('malformed response')); + + $outcomes = app(CategorizeTransactions::class)->forTransactions($user, collect([$transaction])); + + expect($outcomes)->toBe([]); + + Exceptions::assertReported(fn (RuntimeException $e): bool => $e->getMessage() === 'malformed response'); + Queue::assertNotPushed(RetryTransientAiCategorizationJob::class); +}); + it('skips results whose category index does not resolve', function () { $user = User::factory()->create(); groceries($user); diff --git a/tests/Feature/Ai/RetryTransientAiCategorizationJobTest.php b/tests/Feature/Ai/RetryTransientAiCategorizationJobTest.php new file mode 100644 index 00000000..34f82d2c --- /dev/null +++ b/tests/Feature/Ai/RetryTransientAiCategorizationJobTest.php @@ -0,0 +1,76 @@ +create(); + $user->recordAiConsent(); + + $category = Category::factory()->for($user)->create([ + 'type' => CategoryType::Expense, + 'cashflow_direction' => CategoryCashflowDirection::Outflow, + ]); + + $catalog = CategoryCatalog::forUser($user); + $index = 0; + while (($id = $catalog->categoryIdForIndex($index)) !== null && $id !== $category->id) { + $index++; + } + + $transaction = Transaction::factory()->plaintext()->create([ + 'user_id' => $user->id, + 'category_id' => null, + 'category_source' => null, + 'amount' => -4300, + 'creditor_name' => 'mercadona', + 'description' => 'mercadona compra', + ]); + + TransactionCategorizationAgent::fake([ + ['results' => [[ + 'ref' => $transaction->id, + 'category_index' => $index, + 'confidence' => 0.95, + 'merchant_unambiguous' => true, + ]]], + ]); + + app()->call([new RetryTransientAiCategorizationJob($user), 'handle']); + + $transaction->refresh(); + + expect($transaction->category_id)->toBe($category->id) + ->and($transaction->category_source)->toBe(CategorySource::Ai); +}); + +it('does nothing when the user has not consented to AI', function () { + $user = User::factory()->create(); + + Category::factory()->for($user)->create([ + 'type' => CategoryType::Expense, + 'cashflow_direction' => CategoryCashflowDirection::Outflow, + ]); + + $transaction = Transaction::factory()->plaintext()->create([ + 'user_id' => $user->id, + 'category_id' => null, + 'creditor_name' => 'mercadona', + ]); + + TransactionCategorizationAgent::fake([])->preventStrayPrompts(); + + app()->call([new RetryTransientAiCategorizationJob($user), 'handle']); + + $transaction->refresh(); + + expect($transaction->category_id)->toBeNull(); + TransactionCategorizationAgent::assertNeverPrompted(); +});