From d48fea15b2c48e4f4647d7762569774d42e3a87d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Falc=C3=B3n?= Date: Mon, 23 Feb 2026 15:10:40 +0100 Subject: [PATCH] perf: make banking syncs incremental on subsequent runs (#141) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Subsequent syncs (every 6h) now only process recent data instead of re-syncing full history, reducing unnecessary API calls and database writes - Full sync still runs automatically on first connection and can be forced anytime with `banking:sync --full` - Centralizes `isFirstSync` logic in `SyncBankingConnectionJob` and propagates the `fullSync` flag through the entire chain: Command → `SyncAllBankingConnectionsJob` → `SyncBankingConnectionJob` → provider services ## Changes by provider - **Indexa Capital**: Skips portfolio entries older than the last recorded balance date on incremental syncs (the API doesn't support date filtering, so filtering is done client-side) - **Binance**: Reuses stored `invested_amount` from the database on subsequent syncs instead of fetching up to 2 years of deposit/withdrawal history in 90-day windows - **EnableBanking / Bitpanda**: Already minimal — no changes needed ## Testing - Fixed 6 existing Binance tests to pass `isFirstSync: true` for invested amount calculation - Added 7 new tests covering incremental sync behavior, full sync override, and `--full` flag propagation --- .../Commands/SyncBankingConnections.php | 12 ++- app/Jobs/SyncAllBankingConnectionsJob.php | 6 +- app/Jobs/SyncBankingConnectionJob.php | 19 ++-- .../Banking/BinanceBalanceSyncService.php | 20 +++- .../IndexaCapitalBalanceSyncService.php | 20 +++- .../OpenBanking/BinanceBalanceSyncTest.php | 54 +++++++++-- .../IndexaCapitalBalanceSyncTest.php | 96 +++++++++++++++++++ .../SyncBankingConnectionJobTest.php | 26 ++++- .../SyncBankingConnectionsCommandTest.php | 37 +++++++ 9 files changed, 264 insertions(+), 26 deletions(-) diff --git a/app/Console/Commands/SyncBankingConnections.php b/app/Console/Commands/SyncBankingConnections.php index 849b8edd..59ea05a6 100644 --- a/app/Console/Commands/SyncBankingConnections.php +++ b/app/Console/Commands/SyncBankingConnections.php @@ -14,7 +14,8 @@ class SyncBankingConnections extends Command protected $signature = 'banking:sync {--user= : Filter by user email address} {--connection= : Filter by banking connection ID} - {--sync : Run synchronously instead of dispatching to the queue}'; + {--sync : Run synchronously instead of dispatching to the queue} + {--full : Force a full sync instead of incremental}'; protected $description = 'Sync transactions and balances for all active banking connections'; @@ -23,6 +24,7 @@ class SyncBankingConnections extends Command $userEmail = $this->option('user'); $connectionId = $this->option('connection'); $sync = $this->option('sync'); + $fullSync = $this->option('full'); if (! $userEmail && ! $connectionId) { if ($sync) { @@ -31,7 +33,7 @@ class SyncBankingConnections extends Command return Command::FAILURE; } - SyncAllBankingConnectionsJob::dispatch(); + SyncAllBankingConnectionsJob::dispatch($fullSync); $this->info('Banking sync jobs dispatched for all active connections.'); @@ -69,13 +71,13 @@ class SyncBankingConnections extends Command return Command::SUCCESS; } - $connections->each(function (BankingConnection $connection) use ($sync) { + $connections->each(function (BankingConnection $connection) use ($sync, $fullSync) { if ($sync) { $this->info("Syncing {$connection->provider} connection {$connection->id}..."); - SyncBankingConnectionJob::dispatchSync($connection); + SyncBankingConnectionJob::dispatchSync($connection, $fullSync); $this->info("Finished syncing {$connection->provider} connection {$connection->id}."); } else { - SyncBankingConnectionJob::dispatch($connection); + SyncBankingConnectionJob::dispatch($connection, $fullSync); } }); diff --git a/app/Jobs/SyncAllBankingConnectionsJob.php b/app/Jobs/SyncAllBankingConnectionsJob.php index 51fa043a..4250004c 100644 --- a/app/Jobs/SyncAllBankingConnectionsJob.php +++ b/app/Jobs/SyncAllBankingConnectionsJob.php @@ -14,6 +14,10 @@ class SyncAllBankingConnectionsJob implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + public function __construct( + public bool $fullSync = false, + ) {} + public function handle(): void { BankingConnection::query() @@ -23,7 +27,7 @@ class SyncAllBankingConnectionsJob implements ShouldQueue ->orWhere('valid_until', '>', now()); }) ->each(function (BankingConnection $connection) { - SyncBankingConnectionJob::dispatch($connection); + SyncBankingConnectionJob::dispatch($connection, $this->fullSync); }); } } diff --git a/app/Jobs/SyncBankingConnectionJob.php b/app/Jobs/SyncBankingConnectionJob.php index 8beb9bc9..33e4d995 100644 --- a/app/Jobs/SyncBankingConnectionJob.php +++ b/app/Jobs/SyncBankingConnectionJob.php @@ -33,6 +33,7 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue public function __construct( public BankingConnection $bankingConnection, + public bool $fullSync = false, ) {} public function uniqueId(): string @@ -56,14 +57,16 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue } try { + $isFirstSync = ! $connection->last_synced_at || $this->fullSync; + if ($connection->isIndexaCapital()) { - $this->syncIndexaCapital($connection); + $this->syncIndexaCapital($connection, $isFirstSync); } elseif ($connection->isBinance()) { - $this->syncBinance($connection); + $this->syncBinance($connection, $isFirstSync); } elseif ($connection->isBitpanda()) { $this->syncBitpanda($connection); } else { - $this->syncEnableBanking($connection, $transactionSync, $balanceSync); + $this->syncEnableBanking($connection, $transactionSync, $balanceSync, $isFirstSync); } $connection->update([ @@ -85,7 +88,7 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue } } - private function syncIndexaCapital(BankingConnection $connection): void + private function syncIndexaCapital(BankingConnection $connection, bool $isFirstSync): void { $client = new IndexaCapitalClient($connection->api_token); $syncService = new IndexaCapitalBalanceSyncService; @@ -93,13 +96,12 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue $connection->load('accounts'); foreach ($connection->accounts as $account) { - $syncService->sync($account, $client); + $syncService->sync($account, $client, $isFirstSync); } } - private function syncBinance(BankingConnection $connection): void + private function syncBinance(BankingConnection $connection, bool $isFirstSync): void { - $isFirstSync = ! $connection->last_synced_at; $client = new BinanceClient($connection->api_token, $connection->api_secret); $syncService = app(BinanceBalanceSyncService::class); @@ -127,9 +129,8 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue } } - private function syncEnableBanking(BankingConnection $connection, TransactionSyncService $transactionSync, BalanceSyncService $balanceSync): void + private function syncEnableBanking(BankingConnection $connection, TransactionSyncService $transactionSync, BalanceSyncService $balanceSync, bool $isFirstSync): void { - $isFirstSync = ! $connection->last_synced_at; $dateFrom = $isFirstSync ? now()->subYear()->toDateString() : $connection->last_synced_at->toDateString(); diff --git a/app/Services/Banking/BinanceBalanceSyncService.php b/app/Services/Banking/BinanceBalanceSyncService.php index 8c80185f..4ce85507 100644 --- a/app/Services/Banking/BinanceBalanceSyncService.php +++ b/app/Services/Banking/BinanceBalanceSyncService.php @@ -53,9 +53,14 @@ class BinanceBalanceSyncService Sleep::for(self::THROTTLE_SECONDS)->seconds(); } - $investedAmountCents = $this->calculateInvestedAmount($account, $client); + $investedAmountCents = null; - Sleep::for(self::THROTTLE_SECONDS)->seconds(); + if ($isFirstSync) { + $investedAmountCents = $this->calculateInvestedAmount($account, $client); + Sleep::for(self::THROTTLE_SECONDS)->seconds(); + } else { + $investedAmountCents = $this->getLastInvestedAmount($account); + } $this->syncCurrentBalance($account, $client, $investedAmountCents); } @@ -293,6 +298,17 @@ class BinanceBalanceSyncService return 0.0; } + /** + * Get the last stored invested amount for the account. + */ + private function getLastInvestedAmount(Account $account): ?int + { + return $account->balances() + ->whereNotNull('invested_amount') + ->latest('balance_date') + ->value('invested_amount'); + } + /** * Calculate the net invested amount by fetching all deposit and withdrawal history. * Net invested = sum of completed deposits - sum of completed withdrawals, converted to fiat. diff --git a/app/Services/Banking/IndexaCapitalBalanceSyncService.php b/app/Services/Banking/IndexaCapitalBalanceSyncService.php index 2df9048a..118d00f1 100644 --- a/app/Services/Banking/IndexaCapitalBalanceSyncService.php +++ b/app/Services/Banking/IndexaCapitalBalanceSyncService.php @@ -9,9 +9,10 @@ class IndexaCapitalBalanceSyncService { /** * Sync portfolio balances for an Indexa Capital account. - * Stores up to one year of daily historical balances from the portfolios data. + * On first sync, stores all available daily historical balances. + * On subsequent syncs, only processes entries since the last recorded balance. */ - public function sync(Account $account, IndexaCapitalClient $client): void + public function sync(Account $account, IndexaCapitalClient $client, bool $isFirstSync = true): void { if (! $account->external_account_id) { return; @@ -29,6 +30,16 @@ class IndexaCapitalBalanceSyncService return; } + $sinceDate = null; + + if (! $isFirstSync) { + $lastBalanceDate = $account->balances()->max('balance_date'); + + if ($lastBalanceDate) { + $sinceDate = $lastBalanceDate; + } + } + $count = 0; foreach ($portfolios as $entry) { @@ -39,6 +50,10 @@ class IndexaCapitalBalanceSyncService continue; } + if ($sinceDate !== null && $date < $sinceDate) { + continue; + } + $balanceCents = (int) round(floatval($value) * 100); $investedAmountCents = $this->calculateInvestedAmount($entry); @@ -56,6 +71,7 @@ class IndexaCapitalBalanceSyncService Log::info('Synced Indexa Capital balances', [ 'account_id' => $account->id, 'days_synced' => $count, + ...($sinceDate ? ['since_date' => $sinceDate] : []), ]); } diff --git a/tests/Feature/OpenBanking/BinanceBalanceSyncTest.php b/tests/Feature/OpenBanking/BinanceBalanceSyncTest.php index 9ed55b78..1b6fb09c 100644 --- a/tests/Feature/OpenBanking/BinanceBalanceSyncTest.php +++ b/tests/Feature/OpenBanking/BinanceBalanceSyncTest.php @@ -502,7 +502,7 @@ test('calculates invested_amount from deposit and withdrawal history', function $client = new BinanceClient('test-key', 'test-secret'); $service = app(BinanceBalanceSyncService::class); - $service->sync($account, $client); + $service->sync($account, $client, isFirstSync: true); $balance = $account->balances()->first(); // Deposits: 0.5 BTC → converted via CurrencyConversionService (0.5 / 0.00002 = 25000 EUR) @@ -560,7 +560,7 @@ test('excludes internal transfers from invested_amount calculation', function () $client = new BinanceClient('test-key', 'test-secret'); $service = app(BinanceBalanceSyncService::class); - $service->sync($account, $client); + $service->sync($account, $client, isFirstSync: true); $balance = $account->balances()->first(); // Only the external deposit of 500 EUR should count, internal 1000 EUR is excluded @@ -631,7 +631,7 @@ test('filters deposits by status 1 and withdrawals by status 6', function () { $client = new BinanceClient('test-key', 'test-secret'); $service = app(BinanceBalanceSyncService::class); - $service->sync($account, $client); + $service->sync($account, $client, isFirstSync: true); $balance = $account->balances()->first(); // Deposits: 1000 EUR (completed) — pending 2000 excluded @@ -730,7 +730,7 @@ test('converts stablecoin deposits to fiat for invested_amount', function () { $client = new BinanceClient('test-key', 'test-secret'); $service = app(BinanceBalanceSyncService::class); - $service->sync($account, $client); + $service->sync($account, $client, isFirstSync: true); $balance = $account->balances()->first(); // Stablecoins USDT and USDC are treated as USD @@ -790,7 +790,7 @@ test('paginates within a window when deposit history hits the 1000-record limit' $client = new BinanceClient('test-key', 'test-secret'); $service = app(BinanceBalanceSyncService::class); - $service->sync($account, $client); + $service->sync($account, $client, isFirstSync: true); $balance = $account->balances()->first(); // 1000 deposits of 1 EUR + 1 deposit of 500 EUR = 1500 EUR → 150000 cents @@ -836,9 +836,51 @@ test('fetches deposits from older windows when recent window is empty', function $client = new BinanceClient('test-key', 'test-secret'); $service = app(BinanceBalanceSyncService::class); - $service->sync($account, $client); + $service->sync($account, $client, isFirstSync: true); $balance = $account->balances()->first(); // The deposit from the older window should be found: 1000 EUR → 100000 cents expect($balance->invested_amount)->toBe(100000); }); + +test('subsequent sync reuses last invested_amount instead of recalculating', function () { + $user = User::factory()->onboarded()->create(['currency_code' => 'EUR']); + $connection = BankingConnection::factory()->binance()->create([ + 'user_id' => $user->id, + ]); + $account = Account::factory()->connected()->create([ + 'user_id' => $user->id, + 'banking_connection_id' => $connection->id, + 'external_account_id' => 'binance-portfolio', + 'currency_code' => 'EUR', + ]); + + // Pre-existing balance with invested_amount from a previous first sync + $account->balances()->create([ + 'balance_date' => now()->subDay()->toDateString(), + 'balance' => 5000000, + 'invested_amount' => 300000, // 3000 EUR + ]); + + Http::fake([ + // No deposit/withdrawal API calls should be made on subsequent sync + 'api.binance.com/sapi/v1/accountSnapshot*' => Http::response(['snapshotVos' => []]), + 'api.binance.com/api/v3/account*' => Http::response([ + 'balances' => [ + ['asset' => 'BTC', 'free' => '1.0', 'locked' => '0.0'], + ], + ]), + 'api.binance.com/api/v3/ticker/price' => Http::response([ + ['symbol' => 'BTCEUR', 'price' => '50000.00'], + ]), + ]); + + $client = new BinanceClient('test-key', 'test-secret'); + $service = app(BinanceBalanceSyncService::class); + $service->sync($account, $client, isFirstSync: false); + + $todayBalance = $account->balances()->where('balance_date', now()->toDateString())->first(); + expect($todayBalance->balance)->toBe(5000000); + // Should carry forward the last invested_amount + expect($todayBalance->invested_amount)->toBe(300000); +}); diff --git a/tests/Feature/OpenBanking/IndexaCapitalBalanceSyncTest.php b/tests/Feature/OpenBanking/IndexaCapitalBalanceSyncTest.php index cb91753e..e8f5db9c 100644 --- a/tests/Feature/OpenBanking/IndexaCapitalBalanceSyncTest.php +++ b/tests/Feature/OpenBanking/IndexaCapitalBalanceSyncTest.php @@ -271,3 +271,99 @@ test('falls back to total_amount minus return when instruments_cost is missing', // invested_amount = total_amount - return = 8000 - (-500) = 8500 → 850000 cents expect($balance->invested_amount)->toBe(850000); }); + +test('subsequent sync only processes entries since last balance date', function () { + $user = User::factory()->onboarded()->create(); + $connection = BankingConnection::factory()->indexaCapital()->create([ + 'user_id' => $user->id, + ]); + $account = Account::factory()->connected()->create([ + 'user_id' => $user->id, + 'banking_connection_id' => $connection->id, + 'external_account_id' => 'IC-001', + ]); + + // Simulate existing balances from a previous full sync + $account->balances()->create([ + 'balance_date' => now()->subDays(5)->toDateString(), + 'balance' => 1400000, + ]); + $account->balances()->create([ + 'balance_date' => now()->subDays(4)->toDateString(), + 'balance' => 1410000, + ]); + + Http::fake([ + 'api.indexacapital.com/accounts/IC-001/performance' => Http::response([ + 'portfolios' => [ + // Old entries that should be skipped + ['date' => now()->subDays(10)->toDateString(), 'total_amount' => 13000.00], + ['date' => now()->subDays(6)->toDateString(), 'total_amount' => 13900.00], + // Entry on the last balance date (should be processed — updated) + ['date' => now()->subDays(4)->toDateString(), 'total_amount' => 14200.00], + // New entries + ['date' => now()->subDays(3)->toDateString(), 'total_amount' => 14500.00], + ['date' => now()->subDays(2)->toDateString(), 'total_amount' => 14800.00], + ['date' => now()->toDateString(), 'total_amount' => 15000.00], + ], + ]), + ]); + + $client = new IndexaCapitalClient('test-token'); + $service = new IndexaCapitalBalanceSyncService; + $service->sync($account, $client, isFirstSync: false); + + // 2 pre-existing + 3 new entries = 5 total (the one on the boundary date gets updated, not duplicated) + expect($account->balances()->count())->toBe(5); + + // Verify old entry was NOT overwritten (the one from subDays(5) should still be there) + $oldBalance = $account->balances()->where('balance_date', now()->subDays(5)->toDateString())->first(); + expect($oldBalance->balance)->toBe(1400000); + + // Verify boundary entry was updated + $boundaryBalance = $account->balances()->where('balance_date', now()->subDays(4)->toDateString())->first(); + expect($boundaryBalance->balance)->toBe(1420000); + + // Verify new entry was created + $newBalance = $account->balances()->where('balance_date', now()->toDateString())->first(); + expect($newBalance->balance)->toBe(1500000); +}); + +test('full sync processes all entries regardless of existing balances', function () { + $user = User::factory()->onboarded()->create(); + $connection = BankingConnection::factory()->indexaCapital()->create([ + 'user_id' => $user->id, + ]); + $account = Account::factory()->connected()->create([ + 'user_id' => $user->id, + 'banking_connection_id' => $connection->id, + 'external_account_id' => 'IC-001', + ]); + + // Simulate existing balances from a previous sync + $account->balances()->create([ + 'balance_date' => now()->subDays(2)->toDateString(), + 'balance' => 1400000, + ]); + + Http::fake([ + 'api.indexacapital.com/accounts/IC-001/performance' => Http::response([ + 'portfolios' => [ + ['date' => now()->subDays(5)->toDateString(), 'total_amount' => 13000.00], + ['date' => now()->subDays(2)->toDateString(), 'total_amount' => 14000.00], + ['date' => now()->toDateString(), 'total_amount' => 15000.00], + ], + ]), + ]); + + $client = new IndexaCapitalClient('test-token'); + $service = new IndexaCapitalBalanceSyncService; + $service->sync($account, $client, isFirstSync: true); + + // All 3 entries processed (1 existing updated + 2 new) + expect($account->balances()->count())->toBe(3); + + // The old entry at subDays(2) should be updated with new value + $updatedBalance = $account->balances()->where('balance_date', now()->subDays(2)->toDateString())->first(); + expect($updatedBalance->balance)->toBe(1400000); +}); diff --git a/tests/Feature/OpenBanking/SyncBankingConnectionJobTest.php b/tests/Feature/OpenBanking/SyncBankingConnectionJobTest.php index 75165190..7dde25a6 100644 --- a/tests/Feature/OpenBanking/SyncBankingConnectionJobTest.php +++ b/tests/Feature/OpenBanking/SyncBankingConnectionJobTest.php @@ -433,6 +433,7 @@ test('binance first sync gets current balance immediately and dispatches histori }); test('binance subsequent sync does not dispatch historical job', function () { + Mail::fake(); Queue::fake(SyncBinanceHistoricalBalancesJob::class); $user = User::factory()->onboarded()->create(['currency_code' => 'EUR']); @@ -467,7 +468,30 @@ test('binance subsequent sync does not dispatch historical job', function () { $job = new SyncBankingConnectionJob($connection); $job->handle($transactionSync, $balanceSync); - Queue::assertNotPushed(SyncBinanceHistoricalBalancesJob::class); + Mail::assertNothingQueued(); +}); + +test('fullSync flag forces first-sync behavior on already-synced connection', function () { + $user = User::factory()->onboarded()->create(); + $connection = BankingConnection::factory()->create([ + 'user_id' => $user->id, + 'last_synced_at' => now()->subDay(), + ]); + $account = Account::factory()->connected()->create([ + 'user_id' => $user->id, + 'banking_connection_id' => $connection->id, + 'external_account_id' => 'ext-123', + ]); + + $transactionSync = Mockery::mock(TransactionSyncService::class); + $transactionSync->shouldReceive('sync')->once()->andReturn(0); + + $balanceSync = Mockery::mock(BalanceSyncService::class); + $balanceSync->shouldReceive('sync')->once(); + $balanceSync->shouldReceive('calculateHistoricalBalances')->once(); + + $job = new SyncBankingConnectionJob($connection, fullSync: true); + $job->handle($transactionSync, $balanceSync); }); test('bitpanda sync calls balance sync service and updates last_synced_at', function () { diff --git a/tests/Feature/SyncBankingConnectionsCommandTest.php b/tests/Feature/SyncBankingConnectionsCommandTest.php index 0b1a7261..fd422a85 100644 --- a/tests/Feature/SyncBankingConnectionsCommandTest.php +++ b/tests/Feature/SyncBankingConnectionsCommandTest.php @@ -103,3 +103,40 @@ test('banking:sync can combine user and connection filters', function () { return $job->bankingConnection->id === $connection->id; }); }); + +test('banking:sync passes fullSync flag when --full is provided', function () { + Queue::fake(); + + artisan('banking:sync', ['--full' => true]) + ->expectsOutputToContain('Banking sync jobs dispatched for all active connections.') + ->assertSuccessful(); + + Queue::assertPushed(SyncAllBankingConnectionsJob::class, function ($job) { + return $job->fullSync === true; + }); +}); + +test('banking:sync passes fullSync to individual connections with --full', function () { + Queue::fake(); + + $user = User::factory()->create(['email' => 'test@example.com']); + BankingConnection::factory()->for($user)->create(); + + artisan('banking:sync', ['--user' => 'test@example.com', '--full' => true]) + ->assertSuccessful(); + + Queue::assertPushed(SyncBankingConnectionJob::class, function ($job) { + return $job->fullSync === true; + }); +}); + +test('banking:sync does not set fullSync by default', function () { + Queue::fake(); + + artisan('banking:sync') + ->assertSuccessful(); + + Queue::assertPushed(SyncAllBankingConnectionsJob::class, function ($job) { + return $job->fullSync === false; + }); +});