bankingConnection->id; } public function handle(BankingConnectionSyncerFactory $syncerFactory): void { $connection = $this->bankingConnection; $startTime = microtime(true); $syncedAt = now(); $connection->loadMissing('user'); $this->setSentryContext($connection); if (! $connection->user) { Log::info('Banking connection belongs to deleted user, skipping sync', ['connection_id' => $connection->id]); $this->logSyncAttempt($connection, BankingSyncLogStatus::Skipped, $startTime, metadata: ['reason' => 'deleted_user']); return; } $syncer = $syncerFactory->make($connection); if ($syncer->expires() && $connection->isExpired()) { $this->markExpired($connection, $startTime); return; } if (! $this->isSyncableStatus($connection)) { $this->logSyncAttempt($connection, BankingSyncLogStatus::Skipped, $startTime, metadata: ['reason' => 'not_syncable', 'status' => $connection->status->value]); return; } if ($connection->isRateLimited()) { Log::info('Banking connection rate limited, skipping sync', [ 'connection_id' => $connection->id, 'rate_limited_until' => $connection->rate_limited_until?->toIso8601String(), ]); $this->logSyncAttempt($connection, BankingSyncLogStatus::Skipped, $startTime, metadata: [ 'reason' => 'rate_limited', 'rate_limited_until' => $connection->rate_limited_until?->toIso8601String(), ]); return; } try { $isFirstSync = ! $connection->last_synced_at || $this->fullSync; $metadata = $syncer->sync($connection, $isFirstSync); $connection->update([ 'status' => BankingConnectionStatus::Active, 'last_synced_at' => $syncedAt, 'error_message' => null, 'rate_limited_until' => null, 'consecutive_sync_failures' => 0, ]); $this->logSyncAttempt($connection, BankingSyncLogStatus::Success, $startTime, metadata: $metadata ?: null); } catch (ExpiredBankingSessionException) { $this->markExpired($connection, $startTime); return; } catch (\Throwable $e) { $context = [ 'connection_id' => $connection->id, 'error' => $e->getMessage(), 'attempt' => $this->attempts(), ]; if ($e instanceof TransientBankingProviderException) { $context['provider'] = $e->provider; $context['status_code'] = $e->statusCode; $context['provider_code'] = $e->providerCode; } // Only report once the connection actually gives up. Transient errors on a // non-final attempt are recovered by the retry and would otherwise spam one // warning per scheduled cycle for connections that ultimately sync fine. if ($this->attempts() >= $this->tries || $this->isAuthError($e)) { Log::log($e instanceof TransientBankingProviderException ? 'warning' : 'error', 'Banking sync failed', $context); } if ($this->isRateLimitError($e)) { $this->applyRateLimitBackoff($connection, $e); $this->logSyncAttempt($connection, BankingSyncLogStatus::Failed, $startTime, $e); return; } $this->logSyncAttempt($connection, BankingSyncLogStatus::Failed, $startTime, $e); if ($this->isAuthError($e)) { $this->handlePermanentError($connection, $syncer, $e); return; } $this->handleTemporaryError($connection, $e); } } /** * Last resort for a job that died outside handle()'s own error handling. * * Everything raised inside handle() is already classified and recorded, and * leaves the connection in Error - which the status guard below returns on, * after the death itself has been recorded. So what reaches here is the job * being killed from the outside: the queue worker's timeout, an exhausted * retry count, the worker being restarted mid-sync. * * None of that is evidence the *connection* is broken, so it must not be * charged to the connection's budget of scheduled retries. That budget is * what keeps it in the scheduled rotation at all, and spending it here means * spending it on our own infrastructure. * * Note the guard makes this at most one increment per connection lifetime - * a second out-of-band death finds the connection already in Error and does * nothing - so removing it is a small correction, not a fix for a runaway * counter. It closes the one route that could reach the ceiling this way: * a reconnect that left the counter at MAX - 1 (see AuthorizationController) * followed by a single job death. */ public function failed(?\Throwable $e): void { $connection = $this->bankingConnection->fresh(); if (! $connection) { return; } // Recorded before the status guards, not after them. handle() owns every // other logSyncAttempt call and an out-of-band death skips all of them, so // this is the connection's only trace of the most common way this job dies // - and the deaths that repeat are exactly the ones the guards drop. The // connection this was written for is already parked in Error and stays // there: 68 failed jobs against 3 sync-log rows, and logging after the // guard would have added none of the missing 65. $this->logSyncAttempt( $connection, BankingSyncLogStatus::Failed, startTime: null, error: $e, metadata: ['reason' => 'job_died_outside_handle'], ); if ($connection->status === BankingConnectionStatus::Error || ! $this->isSyncableStatus($connection)) { return; } $connection->update([ 'status' => BankingConnectionStatus::Error, // Every message written here describes an out-of-band death, so the // generic "an unexpected error occurred, please try again" was pushing // our own infrastructure onto the user. The next scheduled cycle picks // the connection back up on its own. 'error_message' => __('The sync did not finish. We will try again later.'), ]); } /** * Mark the connection as expired and notify the user to reconnect. * * Reached both when the stored consent window lapses and when the provider * reports the session itself has expired mid-sync. Either way it is an * expected lifecycle event, not a failure to report. */ private function markExpired(BankingConnection $connection, float $startTime): void { $shouldNotify = $connection->status !== BankingConnectionStatus::Expired; $connection->update(['status' => BankingConnectionStatus::Expired]); Log::info('Banking connection expired, skipping sync', ['connection_id' => $connection->id]); if ($shouldNotify && $connection->user?->canReceiveEmails()) { Mail::to($connection->user)->send(new BankingConnectionExpiredEmail( $connection->user, $connection, )); } $this->logSyncAttempt($connection, BankingSyncLogStatus::Skipped, $startTime, metadata: ['reason' => 'expired']); } private function handlePermanentError(BankingConnection $connection, BankingConnectionSyncer $syncer, \Throwable $e): void { $connection->update([ 'status' => BankingConnectionStatus::Error, 'error_message' => $this->friendlyErrorMessage($e), 'consecutive_sync_failures' => self::MAX_SCHEDULED_RETRIES + 1, ]); if ($syncer->notifiesOnAuthFailure() && $connection->user?->canReceiveEmails()) { Mail::to($connection->user)->send(new BankingConnectionAuthFailedEmail( $connection->user, $connection, )); } $this->fail($e); throw $e; } /** * Handle temporary errors that may resolve on retry. * * A provider outage must not spend the connection's budget of scheduled * retries: MAX_SCHEDULED_RETRIES failures drop it out of every future * scheduled sync, silently and with no way back other than reconnecting. * Reaching that state because the bank was down for three cycles is the * wrong trade, so a classified-transient failure surfaces on the * connection without moving the counter. * * ponytail: a provider that is down forever is then retried forever. That * costs one job per cycle and shows up as a repeating warning; add a cap * if the retries ever become expensive. */ private function handleTemporaryError(BankingConnection $connection, \Throwable $e): void { $isFinalAttempt = $this->attempts() >= $this->tries; if ($isFinalAttempt) { $isTransient = $e instanceof TransientBankingProviderException; $connection->update([ 'status' => BankingConnectionStatus::Error, 'error_message' => $this->friendlyErrorMessage($e), 'consecutive_sync_failures' => $isTransient ? $connection->consecutive_sync_failures : $connection->consecutive_sync_failures + 1, ]); } throw $e; } /** * Whether the connection status allows syncing. * Allows both Active and Error (for auto-retry from scheduled runs). */ private function isSyncableStatus(BankingConnection $connection): bool { return in_array($connection->status, [ BankingConnectionStatus::Active, BankingConnectionStatus::Error, ]); } private function setSentryContext(BankingConnection $connection): void { configureScope(function (Scope $scope) use ($connection): void { $scope->setTag('banking_connection_id', (string) $connection->id); $scope->setContext('banking_connection', [ 'id' => $connection->id, 'provider' => $connection->provider->value, 'status' => $connection->status->value, ]); if ($connection->user === null) { return; } $scope->setUser([ 'id' => (string) $connection->user->getAuthIdentifier(), 'email' => $connection->user->email, ]); }); } /** * @param float|null $startTime Null when the caller never got to start a * timer, so the duration is genuinely unknown * rather than zero. */ private function logSyncAttempt( BankingConnection $connection, BankingSyncLogStatus $status, ?float $startTime, ?\Throwable $error = null, ?array $metadata = null, ): void { BankingSyncLog::create([ 'banking_connection_id' => $connection->id, 'status' => $status, 'attempt' => $this->attempts(), 'error_message' => $error?->getMessage(), 'error_class' => $error ? get_class($error) : null, 'duration_ms' => $startTime === null ? null : (int) round((microtime(true) - $startTime) * 1000), 'metadata' => $metadata, 'created_at' => now(), ]); } private function friendlyErrorMessage(\Throwable $e): string { if ($e instanceof TransientBankingProviderException) { return __('The bank provider is temporarily unavailable. We will try syncing again later.'); } if ($e instanceof RequestException) { $status = $e->response->status(); return match (true) { $status === 429 => __('Rate limit exceeded. Please wait a few minutes and try again.'), $status === 401 || $status === 403 => __('Authentication failed. Your credentials may have expired or been revoked.'), $status >= 500 => __('The provider is experiencing issues. Please try again later.'), default => __('Failed to sync with the provider. Please try again later.'), }; } return __('An unexpected error occurred during sync. Please try again later.'); } private function isRateLimitError(\Throwable $e): bool { return $e instanceof RequestException && $e->response->status() === 429; } /** * Persist a backoff window so the scheduler stops re-dispatching * the same connection until the provider quota resets. */ private function applyRateLimitBackoff(BankingConnection $connection, \Throwable $e): void { $until = $this->resolveRateLimitBackoffUntil($e); $connection->update([ 'rate_limited_until' => $until, 'error_message' => $this->friendlyErrorMessage($e), ]); Log::warning('Banking connection rate limited, backing off', [ 'connection_id' => $connection->id, 'rate_limited_until' => $until->toIso8601String(), ]); } private function resolveRateLimitBackoffUntil(\Throwable $e): Carbon { $now = now(); if ($e instanceof RequestException) { $retryAfter = $e->response->header('Retry-After'); if (is_numeric($retryAfter) && (int) $retryAfter > 0) { return $now->copy()->addSeconds((int) $retryAfter); } $body = $e->response->json(); $message = is_array($body) ? (string) ($body['message'] ?? '') : ''; // Daily PSU consultation limit resets at midnight UTC. if (str_contains(strtolower($message), 'daily')) { return $now->copy()->utc()->addDay()->startOfDay(); } } // Default: back off one hour for consent / generic 429 responses. return $now->copy()->addHour(); } private function isAuthError(\Throwable $e): bool { return $e instanceof RequestException && in_array($e->response->status(), [401, 403]); } }